omq 0.8.0 → 0.10.0

data/lib/omq/routing/xsub.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # XSUB socket routing: like SUB but subscriptions sent as data messages.
+    #
+    # Subscriptions are sent as data frames: \x01 + prefix for subscribe,
+    # \x00 + prefix for unsubscribe.
+    #
+    class XSub
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine            = engine
+        @connections       = []
+        @recv_queue        = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @send_queue        = Async::LimitedQueue.new(engine.options.send_hwm)
+        @tasks             = []
+        @send_pump_started = false
+        @send_pump_idle    = true
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        @send_queue.enqueue(parts)
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+      def send_pump_idle? = @send_pump_idle
+
+      private
+
+      def start_send_pump
+        @send_pump_started = true
+        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            @send_pump_idle = true
+            parts = @send_queue.dequeue
+            @send_pump_idle = false
+            frame = parts.first&.b
+            next if frame.nil? || frame.empty?
+
+            flag   = frame.getbyte(0)
+            prefix = frame.byteslice(1..) || "".b
+
+            case flag
+            when 0x01
+              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix)) }
+            when 0x00
+              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix)) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omq/routing.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/queue"
+require "async/limited_queue"
+
+module OMQ
+  # Routing strategies for each ZMQ socket type.
+  #
+  # Each strategy manages how messages flow between connections and
+  # the socket's send/recv queues.
+  #
+  module Routing
+    # Shared frozen empty binary string to avoid repeated allocations.
+    EMPTY_BINARY = "".b.freeze
+
+    # Drains all available messages from +queue+ into +batch+ without
+    # blocking. Call after the initial blocking dequeue.
+    #
+    # No cap is needed: IO::Stream auto-flushes at 64 KB, so the
+    # write buffer hits the wire naturally under sustained load.
+    # The explicit flush after the batch pushes out the remainder.
+    #
+    # @param queue [Async::LimitedQueue]
+    # @param batch [Array]
+    # @return [void]
+    #
+    def self.drain_send_queue(queue, batch)
+      loop do
+        msg = queue.dequeue(timeout: 0)
+        break unless msg
+        batch << msg
+      end
+    end
+
+    # Returns the routing strategy class for a socket type.
+    #
+    # @param socket_type [Symbol] e.g. :PAIR, :REQ
+    # @return [Class]
+    #
+    def self.for(socket_type)
+      case socket_type
+      when :PAIR   then Pair
+      when :REQ    then Req
+      when :REP    then Rep
+      when :DEALER then Dealer
+      when :ROUTER then Router
+      when :PUB    then Pub
+      when :SUB    then Sub
+      when :XPUB   then XPub
+      when :XSUB   then XSub
+      when :PUSH    then Push
+      when :PULL    then Pull
+      when :CLIENT  then Client
+      when :SERVER  then Server
+      when :RADIO   then Radio
+      when :DISH    then Dish
+      when :SCATTER then Scatter
+      when :GATHER  then Gather
+      when :PEER    then Peer
+      when :CHANNEL then Channel
+      else raise ArgumentError, "unknown socket type: #{socket_type}"
+      end
+    end
+  end
+end

data/lib/omq/scatter_gather.rb

@@ -2,8 +2,8 @@
 
 module OMQ
   class SCATTER < Socket
-    include ZMTP::Writable
-    include ZMTP::SingleFrame
+    include Writable
+    include SingleFrame
 
     def initialize(endpoints = nil, linger: 0, send_hwm: nil, send_timeout: nil)
       _init_engine(:SCATTER, linger: linger, send_hwm: send_hwm, send_timeout: send_timeout)
@@ -12,8 +12,8 @@ module OMQ
   end
 
   class GATHER < Socket
-    include ZMTP::Readable
-    include ZMTP::SingleFrame
+    include Readable
+    include SingleFrame
 
     def initialize(endpoints = nil, linger: 0, recv_hwm: nil, recv_timeout: nil)
       _init_engine(:GATHER, linger: linger, recv_hwm: recv_hwm, recv_timeout: recv_timeout)

data/lib/omq/single_frame.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module OMQ
+  # Mixin that rejects multipart messages.
+  #
+  # All draft socket types (CLIENT, SERVER, RADIO, DISH, SCATTER,
+  # GATHER, PEER, CHANNEL) require single-frame messages for
+  # thread-safe atomic operations.
+  #
+  module SingleFrame
+    def send(message)
+      if message.is_a?(Array) && message.size > 1
+        raise ArgumentError, "#{self.class} does not support multipart messages"
+      end
+      super
+    end
+  end
+end

data/lib/omq/socket.rb

@@ -6,7 +6,7 @@ module OMQ
   # Socket base class.
   #
   class Socket
-    # @return [ZMTP::Options]
+    # @return [Options]
     #
     attr_reader :options
 
@@ -70,8 +70,11 @@ module OMQ
     # @return [void]
     #
     def bind(endpoint)
-      @engine.bind(endpoint)
-      @last_tcp_port = @engine.last_tcp_port
+      ensure_parent_task
+      Reactor.run do
+        @engine.bind(endpoint)
+        @last_tcp_port = @engine.last_tcp_port
+      end
     end
 
 
@@ -81,7 +84,8 @@ module OMQ
     # @return [void]
     #
     def connect(endpoint)
-      @engine.connect(endpoint)
+      ensure_parent_task
+      Reactor.run { @engine.connect(endpoint) }
     end
 
 
@@ -91,7 +95,7 @@ module OMQ
     # @return [void]
     #
     def disconnect(endpoint)
-      @engine.disconnect(endpoint)
+      Reactor.run { @engine.disconnect(endpoint) }
     end
 
 
@@ -101,7 +105,7 @@ module OMQ
     # @return [void]
     #
     def unbind(endpoint)
-      @engine.unbind(endpoint)
+      Reactor.run { @engine.unbind(endpoint) }
     end
 
 
@@ -145,7 +149,7 @@ module OMQ
     # Closes the socket.
     #
     def close
-      @engine.close
+      Reactor.run { @engine.close }
       nil
     end
 
@@ -187,6 +191,15 @@ module OMQ
     end
 
 
+    # Sets the engine's parent task before the first bind or connect.
+    # Must be called OUTSIDE Reactor.run so that non-Async callers
+    # get the IO thread's root task, not an ephemeral work task.
+    #
+    def ensure_parent_task
+      @engine.capture_parent_task
+    end
+
+
     # Connects or binds based on endpoint prefix convention.
     #
     # @param endpoints [String, nil]
@@ -212,13 +225,15 @@ module OMQ
     #
     def _init_engine(socket_type, linger:, send_hwm: nil, recv_hwm: nil,
                      send_timeout: nil, recv_timeout: nil, conflate: false)
-      @options = ZMTP::Options.new(linger: linger)
+      @options = Options.new(linger: linger)
       @options.send_hwm      = send_hwm     if send_hwm
       @options.recv_hwm      = recv_hwm     if recv_hwm
       @options.send_timeout   = send_timeout if send_timeout
       @options.recv_timeout   = recv_timeout if recv_timeout
       @options.conflate       = conflate
-      @engine  = ZMTP::Engine.new(socket_type, @options)
+      @recv_buffer = []
+      @recv_mutex  = Mutex.new
+      @engine      = Engine.new(socket_type, @options)
     end
   end
 end

data/lib/omq/transport/inproc.rb

@@ -0,0 +1,355 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/queue"
+
+module OMQ
+  module Transport
+    # In-process transport.
+    #
+    # Both peers are Ruby backend sockets in the same process (native
+    # ZMQ's inproc registry is separate and unreachable). Messages are
+    # transferred as Ruby arrays — no ZMTP framing, no byte
+    # serialization. String parts are frozen by Writable#send to
+    # prevent shared mutable state without copying.
+    #
+    module Inproc
+      # Socket types that exchange commands (SUBSCRIBE/CANCEL) over inproc.
+      #
+      COMMAND_TYPES = %i[PUB SUB XPUB XSUB RADIO DISH].freeze
+
+      # Global registry of bound inproc endpoints.
+      #
+      @registry = {}
+      @mutex    = Mutex.new
+      @waiters  = Hash.new { |h, k| h[k] = [] }
+
+
+      class << self
+        # Binds an engine to an inproc endpoint.
+        #
+        # @param endpoint [String] e.g. "inproc://my-endpoint"
+        # @param engine [Engine] the owning engine
+        # @return [Listener]
+        # @raise [ArgumentError] if endpoint is already bound
+        #
+        def bind(endpoint, engine)
+          @mutex.synchronize do
+            raise ArgumentError, "endpoint already bound: #{endpoint}" if @registry.key?(endpoint)
+            @registry[endpoint] = engine
+
+            # Wake any pending connects
+            @waiters[endpoint].each { |p| p.resolve(true) }
+            @waiters.delete(endpoint)
+          end
+          Listener.new(endpoint)
+        end
+
+
+        # Connects to a bound inproc endpoint.
+        #
+        # @param endpoint [String] e.g. "inproc://my-endpoint"
+        # @param engine [Engine] the connecting engine
+        # @return [void]
+        #
+        def connect(endpoint, engine)
+          bound_engine = @mutex.synchronize { @registry[endpoint] }
+
+          unless bound_engine
+            # Endpoint not bound yet. Wait with timeout derived from
+            # reconnect_interval. If it doesn't appear, silently return —
+            # matching ZMQ 4.x behavior where inproc connect to an
+            # unbound endpoint succeeds but messages go nowhere.
+            # A background task retries periodically.
+            ri      = engine.options.reconnect_interval
+            timeout = ri.is_a?(Range) ? ri.begin : ri
+            promise = Async::Promise.new
+            @mutex.synchronize { @waiters[endpoint] << promise }
+            unless promise.wait?(timeout: timeout)
+              @mutex.synchronize { @waiters[endpoint].delete(promise) }
+              start_connect_retry(endpoint, engine)
+              return
+            end
+            bound_engine = @mutex.synchronize { @registry[endpoint] }
+          end
+
+          establish_link(engine, bound_engine, endpoint)
+        end
+
+
+        # Removes a bound endpoint from the registry.
+        #
+        # @param endpoint [String]
+        # @return [void]
+        #
+        def unbind(endpoint)
+          @mutex.synchronize { @registry.delete(endpoint) }
+        end
+
+
+        # Resets the registry. Used in tests.
+        #
+        # @return [void]
+        #
+        def reset!
+          @mutex.synchronize do
+            @registry.clear
+            @waiters.clear
+          end
+        end
+
+
+        private
+
+
+        # Wires up a client-server inproc pipe pair after validating
+        # that the two socket types are compatible.
+        #
+        # @param client_engine [Engine] the connecting engine
+        # @param server_engine [Engine] the bound engine
+        # @param endpoint [String] the inproc endpoint name
+        #
+        def establish_link(client_engine, server_engine, endpoint)
+          client_type = client_engine.socket_type
+          server_type = server_engine.socket_type
+
+          unless Protocol::ZMTP::VALID_PEERS[client_type]&.include?(server_type)
+            raise Protocol::ZMTP::Error,
+                  "incompatible socket types: #{client_type} cannot connect to #{server_type}"
+          end
+
+          # Only PUB/SUB-family types exchange commands (SUBSCRIBE/CANCEL)
+          # over inproc. All other types use only the direct recv queue
+          # bypass for data, so no internal queues are needed.
+          needs_commands = COMMAND_TYPES.include?(client_type) ||
+                           COMMAND_TYPES.include?(server_type)
+
+          if needs_commands
+            a_to_b = Async::Queue.new
+            b_to_a = Async::Queue.new
+          end
+
+          client_pipe = DirectPipe.new(
+            send_queue:    needs_commands ? a_to_b : nil,
+            receive_queue: needs_commands ? b_to_a : nil,
+            peer_identity: server_engine.options.identity,
+            peer_type:     server_type.to_s,
+          )
+          server_pipe = DirectPipe.new(
+            send_queue:    needs_commands ? b_to_a : nil,
+            receive_queue: needs_commands ? a_to_b : nil,
+            peer_identity: client_engine.options.identity,
+            peer_type:     client_type.to_s,
+          )
+
+          client_pipe.peer = server_pipe
+          server_pipe.peer = client_pipe
+
+          client_engine.connection_ready(client_pipe, endpoint: endpoint)
+          server_engine.connection_ready(server_pipe, endpoint: endpoint)
+        end
+
+
+        # Spawns a background task that periodically retries
+        # #establish_link until the endpoint appears in the registry.
+        #
+        # @param endpoint [String] the inproc endpoint name
+        # @param engine [Engine] the connecting engine
+        #
+        def start_connect_retry(endpoint, engine)
+          engine.spawn_inproc_retry(endpoint) do |ivl|
+            loop do
+              sleep ivl
+              bound_engine = @mutex.synchronize { @registry[endpoint] }
+              if bound_engine
+                establish_link(engine, bound_engine, endpoint)
+                break
+              end
+            end
+          end
+        end
+      end
+
+      # A bound inproc endpoint handle.
+      #
+      class Listener
+        # @return [String] the bound endpoint
+        #
+        attr_reader :endpoint
+
+
+        # @param endpoint [String]
+        #
+        def initialize(endpoint)
+          @endpoint = endpoint
+        end
+
+
+        # Stops the listener by removing it from the registry.
+        #
+        # @return [void]
+        #
+        def stop
+          Inproc.unbind(@endpoint)
+        end
+      end
+
+      # A direct in-process pipe that transfers Ruby arrays through queues.
+      #
+      # Implements the same interface as Connection so routing strategies
+      # can use it transparently.
+      #
+      # When a routing strategy sets {#direct_recv_queue} on a pipe,
+      # {#send_message} enqueues directly into the peer's recv queue,
+      # bypassing the intermediate pipe queues and the recv pump task.
+      # This reduces inproc from 3 queue hops to 2 (send_queue →
+      # recv_queue), eliminating the internal pipe queue in between.
+      #
+      class DirectPipe
+        # @return [String] peer's socket type
+        #
+        attr_reader :peer_socket_type
+
+
+        # @return [String] peer's identity
+        #
+        attr_reader :peer_identity
+
+
+        # @return [DirectPipe, nil] the other end of this pipe pair
+        #
+        attr_accessor :peer
+
+
+        # @return [Async::LimitedQueue, nil] when set, {#send_message}
+        #   enqueues directly here instead of using the internal queue
+        #
+        attr_reader :direct_recv_queue
+
+
+        # @return [Proc, nil] optional transform applied before
+        #   enqueuing into {#direct_recv_queue}
+        #
+        attr_accessor :direct_recv_transform
+
+
+        # @param send_queue [Async::Queue, nil] outgoing command queue
+        #   (nil for non-PUB/SUB types that don't exchange commands)
+        # @param receive_queue [Async::Queue, nil] incoming command queue
+        # @param peer_identity [String]
+        # @param peer_type [String]
+        #
+        def initialize(send_queue: nil, receive_queue: nil, peer_identity:, peer_type:)
+          @send_queue            = send_queue
+          @receive_queue         = receive_queue
+          @peer_identity         = peer_identity || "".b
+          @peer_socket_type      = peer_type
+          @closed                = false
+          @peer                  = nil
+          @direct_recv_queue     = nil
+          @direct_recv_transform = nil
+          @pending_direct        = nil
+        end
+
+
+        # Sets the direct recv queue. Drains any messages that were
+        # buffered before the queue was available.
+        #
+        def direct_recv_queue=(queue)
+          @direct_recv_queue = queue
+          if queue && @pending_direct
+            @pending_direct.each do |msg|
+              queue.enqueue(msg)
+            end
+            @pending_direct = nil
+          end
+        end
+
+
+        # Sends a multi-frame message.
+        #
+        # When {#direct_recv_queue} is set (inproc fast path), the
+        # message is delivered directly to the peer's recv queue,
+        # skipping the internal pipe queues and the recv pump.
+        #
+        # @param parts [Array<String>]
+        # @return [void]
+        #
+        def send_message(parts)
+          raise IOError, "closed" if @closed
+          if @direct_recv_queue
+            msg = @direct_recv_transform ? @direct_recv_transform.call(parts).freeze : parts
+            @direct_recv_queue.enqueue(msg)
+          elsif @send_queue
+            @send_queue.enqueue(parts)
+          else
+            msg = @direct_recv_transform ? @direct_recv_transform.call(parts).freeze : parts
+            (@pending_direct ||= []) << msg
+          end
+        end
+
+
+        alias write_message send_message
+
+
+        # No-op — inproc has no IO buffer to flush.
+        #
+        # @return [void]
+        #
+        def flush = nil
+
+
+        # Receives a multi-frame message.
+        #
+        # @return [Array<String>]
+        # @raise [EOFError] if closed
+        #
+        def receive_message
+          msg = @receive_queue.dequeue
+          raise EOFError, "connection closed" if msg.nil?
+          msg
+        end
+
+
+        # Sends a command via the internal command queue.
+        # Only available for PUB/SUB-family pipes.
+        #
+        # @param command [Protocol::ZMTP::Codec::Command]
+        # @return [void]
+        #
+        def send_command(command)
+          raise IOError, "closed" if @closed
+          @send_queue.enqueue([:command, command])
+        end
+
+
+        # Reads one command frame from the internal command queue.
+        # Used by PUB/XPUB subscription listeners.
+        #
+        # @return [Protocol::ZMTP::Codec::Frame]
+        #
+        def read_frame
+          loop do
+            item = @receive_queue.dequeue
+            raise EOFError, "connection closed" if item.nil?
+            if item.is_a?(Array) && item.first == :command
+              cmd = item[1]
+              return Protocol::ZMTP::Codec::Frame.new(cmd.to_body, command: true)
+            end
+          end
+        end
+
+
+        # Closes this pipe end.
+        #
+        # @return [void]
+        #
+        def close
+          return if @closed
+          @closed = true
+          @send_queue&.enqueue(nil) # close sentinel
+        end
+      end
+    end
+  end
+end