omq 0.9.0 → 0.11.0

data/lib/omq/routing/server.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module OMQ
+  module Routing
+    # SERVER socket routing: identity-based routing with auto-generated
+    # 4-byte routing IDs.
+    #
+    # Prepends routing ID on receive. Strips routing ID on send and
+    # routes to the identified connection.
+    #
+    class Server
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine                     = engine
+        @recv_queue                 = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @send_queue                 = Async::LimitedQueue.new(engine.options.send_hwm)
+        @connections_by_routing_id  = {}
+        @routing_id_by_connection   = {}
+        @tasks                      = []
+        @send_pump_started          = false
+        @send_pump_idle             = true
+        @written                    = Set.new
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        routing_id = SecureRandom.bytes(4)
+        @connections_by_routing_id[routing_id] = connection
+        @routing_id_by_connection[connection]  = routing_id
+
+        task = @engine.start_recv_pump(connection, @recv_queue) do |msg|
+          [routing_id, *msg]
+        end
+        @tasks << task if task
+
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        routing_id = @routing_id_by_connection.delete(connection)
+        @connections_by_routing_id.delete(routing_id) if routing_id
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        @send_queue.enqueue(parts)
+      end
+
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+      def send_pump_idle? = @send_pump_idle
+
+      private
+
+      def start_send_pump
+        @send_pump_started = true
+        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            @send_pump_idle = true
+            batch = [@send_queue.dequeue]
+            @send_pump_idle = false
+            Routing.drain_send_queue(@send_queue, batch)
+
+            @written.clear
+            batch.each do |parts|
+              routing_id = parts.first
+              conn       = @connections_by_routing_id[routing_id]
+              next unless conn # silently drop if peer gone
+              begin
+                conn.write_message(parts[1..])
+                @written << conn
+              rescue *CONNECTION_LOST
+                # will be cleaned up
+              end
+            end
+
+            @written.each do |conn|
+              conn.flush
+            rescue *CONNECTION_LOST
+              # will be cleaned up
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omq/routing/sub.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # SUB socket routing: subscription-based receive from PUB peers.
+    #
+    # Sends SUBSCRIBE/CANCEL commands to connected PUB peers.
+    #
+    class Sub
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine        = engine
+        @connections   = []
+        @recv_queue    = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @subscriptions = Set.new
+        @tasks         = []
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        # Send existing subscriptions to new peer
+        @subscriptions.each do |prefix|
+          connection.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
+        end
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+      end
+
+      # SUB is read-only.
+      #
+      def enqueue(_parts)
+        raise "SUB sockets cannot send"
+      end
+
+      # Subscribes to a topic prefix.
+      #
+      # @param prefix [String]
+      #
+      def subscribe(prefix)
+        @subscriptions << prefix
+        @connections.each do |conn|
+          conn.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix))
+        end
+      end
+
+      # Unsubscribes from a topic prefix.
+      #
+      # @param prefix [String]
+      #
+      def unsubscribe(prefix)
+        @subscriptions.delete(prefix)
+        @connections.each do |conn|
+          conn.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix))
+        end
+      end
+
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+    end
+  end
+end

data/lib/omq/routing/xpub.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # XPUB socket routing: like PUB but exposes subscription messages.
+    #
+    # Subscription/unsubscription messages from peers are delivered to
+    # the application as data frames: \x01 + prefix for subscribe,
+    # \x00 + prefix for unsubscribe.
+    #
+    class XPub
+      include FanOut
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine     = engine
+        @recv_queue = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @tasks      = []
+        init_fan_out(engine)
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        @subscriptions[connection] = Set.new
+        start_subscription_listener(connection)
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+        @subscriptions.delete(connection)
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        @send_queue.enqueue(parts)
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+      private
+
+      # Expose subscription to application as data message.
+      #
+      def on_subscribe(conn, prefix)
+        super
+        @recv_queue.enqueue(["\x01#{prefix}".b])
+      end
+
+      # Expose unsubscription to application as data message.
+      #
+      def on_cancel(conn, prefix)
+        super
+        @recv_queue.enqueue(["\x00#{prefix}".b])
+      end
+    end
+  end
+end

data/lib/omq/routing/xsub.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Routing
+    # XSUB socket routing: like SUB but subscriptions sent as data messages.
+    #
+    # Subscriptions are sent as data frames: \x01 + prefix for subscribe,
+    # \x00 + prefix for unsubscribe.
+    #
+    class XSub
+
+      # @param engine [Engine]
+      #
+      def initialize(engine)
+        @engine            = engine
+        @connections       = []
+        @recv_queue        = Async::LimitedQueue.new(engine.options.recv_hwm)
+        @send_queue        = Async::LimitedQueue.new(engine.options.send_hwm)
+        @tasks             = []
+        @send_pump_started = false
+        @send_pump_idle    = true
+      end
+
+      # @return [Async::LimitedQueue]
+      #
+      attr_reader :recv_queue, :send_queue
+
+      # @param connection [Connection]
+      #
+      def connection_added(connection)
+        @connections << connection
+        task = @engine.start_recv_pump(connection, @recv_queue)
+        @tasks << task if task
+        start_send_pump unless @send_pump_started
+      end
+
+      # @param connection [Connection]
+      #
+      def connection_removed(connection)
+        @connections.delete(connection)
+      end
+
+      # @param parts [Array<String>]
+      #
+      def enqueue(parts)
+        @send_queue.enqueue(parts)
+      end
+
+      #
+      def stop
+        @tasks.each(&:stop)
+        @tasks.clear
+      end
+
+      def send_pump_idle? = @send_pump_idle
+
+      private
+
+      def start_send_pump
+        @send_pump_started = true
+        @tasks << @engine.spawn_pump_task(annotation: "send pump") do
+          loop do
+            @send_pump_idle = true
+            parts = @send_queue.dequeue
+            @send_pump_idle = false
+            frame = parts.first&.b
+            next if frame.nil? || frame.empty?
+
+            flag   = frame.getbyte(0)
+            prefix = frame.byteslice(1..) || "".b
+
+            case flag
+            when 0x01
+              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.subscribe(prefix)) }
+            when 0x00
+              @connections.each { |c| c.send_command(Protocol::ZMTP::Codec::Command.cancel(prefix)) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omq/routing.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/queue"
+require "async/limited_queue"
+
+module OMQ
+  # Routing strategies for each ZMQ socket type.
+  #
+  # Each strategy manages how messages flow between connections and
+  # the socket's send/recv queues.
+  #
+  module Routing
+    # Shared frozen empty binary string to avoid repeated allocations.
+    EMPTY_BINARY = "".b.freeze
+
+    # Drains all available messages from +queue+ into +batch+ without
+    # blocking. Call after the initial blocking dequeue.
+    #
+    # No cap is needed: IO::Stream auto-flushes at 64 KB, so the
+    # write buffer hits the wire naturally under sustained load.
+    # The explicit flush after the batch pushes out the remainder.
+    #
+    # @param queue [Async::LimitedQueue]
+    # @param batch [Array]
+    # @return [void]
+    #
+    def self.drain_send_queue(queue, batch)
+      loop do
+        msg = queue.dequeue(timeout: 0)
+        break unless msg
+        batch << msg
+      end
+    end
+
+    # Returns the routing strategy class for a socket type.
+    #
+    # @param socket_type [Symbol] e.g. :PAIR, :REQ
+    # @return [Class]
+    #
+    def self.for(socket_type)
+      case socket_type
+      when :PAIR   then Pair
+      when :REQ    then Req
+      when :REP    then Rep
+      when :DEALER then Dealer
+      when :ROUTER then Router
+      when :PUB    then Pub
+      when :SUB    then Sub
+      when :XPUB   then XPub
+      when :XSUB   then XSub
+      when :PUSH    then Push
+      when :PULL    then Pull
+      when :CLIENT  then Client
+      when :SERVER  then Server
+      when :RADIO   then Radio
+      when :DISH    then Dish
+      when :SCATTER then Scatter
+      when :GATHER  then Gather
+      when :PEER    then Peer
+      when :CHANNEL then Channel
+      else raise ArgumentError, "unknown socket type: #{socket_type}"
+      end
+    end
+  end
+end

data/lib/omq/scatter_gather.rb

@@ -2,21 +2,21 @@
 
 module OMQ
   class SCATTER < Socket
-    include ZMTP::Writable
-    include ZMTP::SingleFrame
+    include Writable
+    include SingleFrame
 
-    def initialize(endpoints = nil, linger: 0, send_hwm: nil, send_timeout: nil)
-      _init_engine(:SCATTER, linger: linger, send_hwm: send_hwm, send_timeout: send_timeout)
+    def initialize(endpoints = nil, linger: 0, send_hwm: nil, send_timeout: nil, backend: nil)
+      _init_engine(:SCATTER, linger: linger, send_hwm: send_hwm, send_timeout: send_timeout, backend: backend)
       _attach(endpoints, default: :connect)
     end
   end
 
   class GATHER < Socket
-    include ZMTP::Readable
-    include ZMTP::SingleFrame
+    include Readable
+    include SingleFrame
 
-    def initialize(endpoints = nil, linger: 0, recv_hwm: nil, recv_timeout: nil)
-      _init_engine(:GATHER, linger: linger, recv_hwm: recv_hwm, recv_timeout: recv_timeout)
+    def initialize(endpoints = nil, linger: 0, recv_hwm: nil, recv_timeout: nil, backend: nil)
+      _init_engine(:GATHER, linger: linger, recv_hwm: recv_hwm, recv_timeout: recv_timeout, backend: backend)
       _attach(endpoints, default: :bind)
     end
   end

data/lib/omq/single_frame.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module OMQ
+  # Mixin that rejects multipart messages.
+  #
+  # All draft socket types (CLIENT, SERVER, RADIO, DISH, SCATTER,
+  # GATHER, PEER, CHANNEL) require single-frame messages for
+  # thread-safe atomic operations.
+  #
+  module SingleFrame
+    def send(message)
+      if message.is_a?(Array) && message.size > 1
+        raise ArgumentError, "#{self.class} does not support multipart messages"
+      end
+      super
+    end
+  end
+end

data/lib/omq/socket.rb

@@ -6,7 +6,7 @@ module OMQ
   # Socket base class.
   #
   class Socket
-    # @return [ZMTP::Options]
+    # @return [Options]
     #
     attr_reader :options
 
@@ -36,7 +36,8 @@ module OMQ
       :heartbeat_ttl,         :heartbeat_ttl=,
       :heartbeat_timeout,     :heartbeat_timeout=,
       :max_message_size,      :max_message_size=,
-      :mechanism,             :mechanism=
+      :mechanism,             :mechanism=,
+      :tls_context,           :tls_context=
 
 
     # Creates a new socket and binds it to the given endpoint.
@@ -70,8 +71,11 @@ module OMQ
     # @return [void]
     #
     def bind(endpoint)
-      @engine.bind(endpoint)
-      @last_tcp_port = @engine.last_tcp_port
+      ensure_parent_task
+      Reactor.run do
+        @engine.bind(endpoint)
+        @last_tcp_port = @engine.last_tcp_port
+      end
     end
 
 
@@ -81,7 +85,8 @@ module OMQ
     # @return [void]
     #
     def connect(endpoint)
-      @engine.connect(endpoint)
+      ensure_parent_task
+      Reactor.run { @engine.connect(endpoint) }
     end
 
 
@@ -91,7 +96,7 @@ module OMQ
     # @return [void]
     #
     def disconnect(endpoint)
-      @engine.disconnect(endpoint)
+      Reactor.run { @engine.disconnect(endpoint) }
     end
 
 
@@ -101,7 +106,7 @@ module OMQ
     # @return [void]
     #
     def unbind(endpoint)
-      @engine.unbind(endpoint)
+      Reactor.run { @engine.unbind(endpoint) }
     end
 
 
@@ -145,7 +150,7 @@ module OMQ
     # Closes the socket.
     #
     def close
-      @engine.close
+      Reactor.run { @engine.close }
       nil
     end
 
@@ -187,6 +192,15 @@ module OMQ
     end
 
 
+    # Sets the engine's parent task before the first bind or connect.
+    # Must be called OUTSIDE Reactor.run so that non-Async callers
+    # get the IO thread's root task, not an ephemeral work task.
+    #
+    def ensure_parent_task
+      @engine.capture_parent_task
+    end
+
+
     # Connects or binds based on endpoint prefix convention.
     #
     # @param endpoints [String, nil]
@@ -211,14 +225,21 @@ module OMQ
     # @param linger [Integer]
     #
     def _init_engine(socket_type, linger:, send_hwm: nil, recv_hwm: nil,
-                     send_timeout: nil, recv_timeout: nil, conflate: false)
-      @options = ZMTP::Options.new(linger: linger)
+                     send_timeout: nil, recv_timeout: nil, conflate: false,
+                     backend: nil)
+      @options = Options.new(linger: linger)
       @options.send_hwm      = send_hwm     if send_hwm
       @options.recv_hwm      = recv_hwm     if recv_hwm
       @options.send_timeout   = send_timeout if send_timeout
       @options.recv_timeout   = recv_timeout if recv_timeout
       @options.conflate       = conflate
-      @engine  = ZMTP::Engine.new(socket_type, @options)
+      @recv_buffer = []
+      @recv_mutex  = Mutex.new
+      @engine      = case backend
+                     when nil, :ruby then Engine.new(socket_type, @options)
+                     when :ffi       then FFI::Engine.new(socket_type, @options)
+                     else raise ArgumentError, "unknown backend: #{backend}"
+                     end
     end
   end
 end