omq 0.28.10 → 0.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f251093afa3c8f3ff07a187f3050535c839dda05f8ccb7dc3f42ea34706fea0
-  data.tar.gz: e1ed93a70c9167bc6df560c57d80411904a3d6fdcaa19ba148887f7f044fb9fa
+  metadata.gz: 0c36b1596063dc355a2c30ad2ee00d5b5646574fbd39aaf7185334f9d2edeacf
+  data.tar.gz: 9fec0824a06493d620b59c808f59864ea43e7f57c0e1a86ef7d0bbcb5a3c0c68
 SHA512:
-  metadata.gz: 8c3b77d4c81dd3b4f7c66decefd86515fcba6fc69683a7f5dc9a829436429d74bc301d65c506b845ff2003e36d9d9c6cbe30f1c665b2baffe7fe31ebe9d7d399
-  data.tar.gz: afc55dd9e6c1c146ba3b948ec1590bfb5df7589c33a528c7646c22b1fb71489e1abf26400b4966afa7caa2777279b12293d5e36ca04c7f029c5b372ddc887b5b
+  metadata.gz: dcc0b56ab0ee65f8e537ecefb3599802f25053bd70ee25acc307a02e34124eafe50ca651010565ec15c3e9b5b8032da480ef458f35b94bd93e3114c444ab435a
+  data.tar.gz: fd213cabc8e6cb194640a474ce1fd4d255a8c959b9110ced374c17c8d2a983f4ed1fd76b0bdb0b5fcb964eb7b654b58c3f6437ac039b0af6e382ebbf7b7bddf3

data/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 ## [Unreleased]
 
+## [0.29.0] - 2026-10-15
+
+### Added
+
+- Shared bounded arena encoding and decoding for the LZ4 and Zstd transports.
+
+### Fixed
+
+- Stop reconnecting with the same credentials after a fatal handshake ERROR.
+- Enforced REQ/REP send/receive order and rejected incomplete routing envelopes.
+- Matched REQ replies to the selected peer and kept REP reply destinations with
+  their requests across disconnects. Replies to muted REP peers are discarded.
+- Applied in-process receive transforms to buffered messages and discarded
+  filtered messages without delivering close sentinels.
+- Removed in-process peer connections when the opposite pipe closes, including
+  connections that bypass the receive pump.
+
+### Removed
+
+- Removed `omq-qos`, its socket options, and its test and release tasks.
+
 ## [0.28.10] - 2026-08-26
 
 ### Changed

data/README.md

@@ -278,16 +278,6 @@ post-handshake message parts are compressed.
   `omq-lz4` for new compressed TCP work. The Rust backend also exposes
   `zstd+tcp://`.
 
-### Protocol extensions (RFCs)
-
-Optional plug-ins that extend the ZMTP wire protocol. Each is a separate gem.
-load the ones you need.
-
-- **[omq-qos](gems/omq-qos)**: experimental MQTT-style at-least-once
-  delivery. Receivers send an ACK command frame keyed by xxHash digest.
-  unacked messages re-enqueue to the next peer on disconnect. PUSH/PULL,
-  SCATTER/GATHER, REQ/REP only. Fan-out patterns are out of scope.
-
 ## Development
 
 ```sh

data/lib/omq/engine/connection_lifecycle.rb

@@ -117,11 +117,12 @@ module OMQ
 
         conn&.close
 
-        # Full tear-down with reconnect: without this, spawn_connection's
-        # ensure-block close! sees :closed and skips maybe_reconnect,
+        # Reconnect after temporary failures. An explicit ERROR response
+        # forbids retrying with the same credentials. Without tear-down,
+        # spawn_connection's ensure-block close! skips maybe_reconnect,
         # leaving the endpoint dead. Race is exposed when a peer RSTs
         # mid-handshake (e.g. LINGER 0 close against an in-flight connect).
-        tear_down!(reconnect: true)
+        tear_down!(reconnect: !error.is_a?(Protocol::ZMTP::FatalHandshakeError))
         raise
       end
 
@@ -132,6 +133,9 @@ module OMQ
       # @param pipe [Transport::Inproc::Pipe]
       #
       def ready_direct!(pipe)
+        if pipe.is_a?(Transport::Inproc::Pipe) && pipe.peer
+          @barrier.async(transient: true, annotation: "inproc peer close") { pipe.peer.wait_closed }
+        end
         ready!(pipe)
       end
 
@@ -186,9 +190,7 @@ module OMQ
         @engine.peer_connected.resolve(@conn)
         transition!(:ready)
 
-        # No supervisor if nothing to supervise: inproc Pipes
-        # wire the recv/send paths synchronously (no task-based pumps),
-        # and isolated unit tests use a FakeEngine without pumps at all.
+        # Isolated unit tests can use a FakeEngine without pumps at all.
         # Waiting on an empty barrier returns immediately and would
         # tear the connection down right after registering.
         start_supervisor unless @barrier.empty?

data/lib/omq/engine/reconnect.rb

@@ -60,6 +60,8 @@ module OMQ
           begin
             @dialer.connect
             break
+          rescue Protocol::ZMTP::FatalHandshakeError
+            break
           rescue *CONNECTION_LOST, *CONNECTION_FAILED, Protocol::ZMTP::Error
             delay = next_delay(delay, max_delay)
             @engine.emit_monitor_event :connect_retried,

data/lib/omq/engine/recv_pump.rb

@@ -48,7 +48,7 @@ module OMQ
         @conn        = conn
         @recv_queue  = recv_queue
         @engine      = engine
-        @count_bytes = conn.instance_of?(Protocol::ZMTP::Connection)
+        @count_bytes = conn.instance_of?(Protocol::ZMTP::Connection) || conn.respond_to?(:last_wire_size_in)
       end
 
 
@@ -83,8 +83,7 @@ module OMQ
       # A transform that returns +nil+ drops the message — the recv pump
       # still counts it toward fairness (so dup-floods can't starve
       # siblings) but it is neither emitted nor enqueued to the
-      # application. omq-qos uses this at QoS >= 2 for dedup-set hits:
-      # the transform ACKs the sender and returns nil.
+      # application.
       #
       # @param parent [Async::Task, Async::Barrier]
       # @param transform [Proc]

data/lib/omq/options.rb

@@ -32,7 +32,6 @@ module OMQ
       @rcvbuf                = nil   # bytes, nil = OS default
       @on_mute               = :block   # :block, :drop_newest, :drop_oldest
       @mechanism             = Protocol::ZMTP::Mechanism::Null.new
-      @qos                   = 0       # 0 = fire-and-forget, 1 = at-least-once (see omq-qos gem)
     end
 
 
@@ -71,8 +70,6 @@ module OMQ
     #   @return [Symbol] mute strategy (:block, :drop_newest, :drop_oldest)
     # @!attribute mechanism
     #   @return [Protocol::ZMTP::Mechanism::Null, Protocol::ZMTP::Mechanism::Curve] security mechanism
-    # @!attribute qos
-    #   @return [Integer] quality of service level (0 = fire-and-forget)
     #
     attr_accessor :send_hwm,  :recv_hwm,
                   :linger,    :identity,
@@ -83,8 +80,7 @@ module OMQ
                   :max_message_size,
                   :sndbuf,    :rcvbuf,
                   :on_mute,
-                  :mechanism,
-                  :qos
+                  :mechanism
 
     # @return [Boolean] true if router_mandatory is set
     alias_method :router_mandatory?, :router_mandatory

data/lib/omq/routing/rep.rb

@@ -20,10 +20,11 @@ module OMQ
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine          = engine
-        @recv_queue      = Routing.build_queue(engine.options.recv_hwm, :block)
-        @pending_replies = []
-        @conn_queues     = {}
+        @engine        = engine
+        @recv_queue    = Routing.build_queue(engine.options.recv_hwm, :block)
+        @current_reply = nil
+        @receiving     = false
+        @conn_queues   = {}
       end
 
 
@@ -32,7 +33,19 @@ module OMQ
       # @return [Array<String>, nil]
       #
       def dequeue_recv
-        @recv_queue.dequeue
+        raise SocketError, "REP socket expects recv/send/recv/send order" if @current_reply || @receiving
+
+        @receiving = true
+        begin
+          request = @recv_queue.dequeue
+          return unless request
+
+          connection, envelope, body = request
+          @current_reply = [connection, envelope]
+          body
+        ensure
+          @receiving = false
+        end
       end
 
 
@@ -48,17 +61,23 @@ module OMQ
       # @param connection [Protocol::ZMTP::Connection]
       #
       def connection_added(connection)
+        q = Routing.build_queue(@engine.options.send_hwm, :drop_newest)
+        @conn_queues[connection] = q
+
         @engine.start_recv_pump(connection, @recv_queue) do |msg|
-          delimiter = msg.index { |p| p.empty? } || msg.size
-          envelope  = msg[0, delimiter]
-          body      = msg[(delimiter + 1)..] || []
+          delimiter = msg.index { |p| p.empty? }
+          next unless delimiter && delimiter < msg.size - 1
 
-          @pending_replies << [connection, envelope]
-          body
+          envelope = msg[0, delimiter].freeze
+          body     = msg[(delimiter + 1)..].freeze
+
+          # Keep the routing information with its body until the application
+          # receives it, even if the peer disconnects in the meantime.
+          @engine.emit_verbose_msg_received(connection, body)
+          @recv_queue.enqueue([connection, envelope, body].freeze)
+          nil
         end
 
-        q = Routing.build_queue(@engine.options.send_hwm, :block)
-        @conn_queues[connection] = q
         ConnSendPump.start(@engine, connection, q)
       end
 
@@ -66,25 +85,22 @@ module OMQ
       # @param connection [Protocol::ZMTP::Connection]
       #
       def connection_removed(connection)
-        @pending_replies.reject! { |r| r[0] == connection }
         @conn_queues.delete(connection)
       end
 
 
       # Enqueues a reply. Routes to the connection that sent the matching
-      # request by consuming the next pending_reply entry.
+      # request most recently received by the application. Replies to
+      # disconnected or muted peers are discarded.
       #
       # @param parts [Array<String>]
       #
       def enqueue(parts)
-        reply_info = @pending_replies.shift
-        return unless reply_info
-
-        conn, envelope = reply_info
-        msg = envelope
-        msg << EMPTY_FRAME
-        msg.concat(parts)
-        @conn_queues[conn]&.enqueue(msg)
+        raise SocketError, "REP socket expects recv/send/recv/send order" unless @current_reply
+
+        conn, envelope = @current_reply
+        @conn_queues[conn]&.enqueue([*envelope, EMPTY_FRAME, *parts].freeze)
+        @current_reply = nil
       end
 
 

data/lib/omq/routing/req.rb

@@ -21,9 +21,11 @@ module OMQ
       # @param engine [Engine]
       #
       def initialize(engine)
-        @engine     = engine
-        @recv_queue = Routing.build_queue(engine.options.recv_hwm, :block)
-        @state      = :ready        # :ready or :waiting_reply
+        @engine           = engine
+        @recv_queue       = Routing.build_queue(engine.options.recv_hwm, :block)
+        @state            = :ready
+        @reply_connection = nil
+        @receiving        = false
         init_round_robin(engine)
       end
 
@@ -33,7 +35,19 @@ module OMQ
       # @return [Array<String>, nil]
       #
       def dequeue_recv
-        @recv_queue.dequeue
+        raise SocketError, "REQ socket expects send/recv/send/recv order" if @state == :ready || @receiving
+
+        @receiving = true
+        begin
+          msg = @recv_queue.dequeue
+          if msg
+            @state            = :ready
+            @reply_connection = nil
+          end
+          msg
+        ensure
+          @receiving = false
+        end
       end
 
 
@@ -50,8 +64,11 @@ module OMQ
       #
       def connection_added(connection)
         @engine.start_recv_pump(connection, @recv_queue) do |msg|
-          @state = :ready
-          msg.first&.empty? ? msg[1..] : msg
+          next unless @state == :waiting_reply && connection.equal?(@reply_connection)
+          next unless msg.size >= 2 && msg.first.empty?
+
+          @state = :reply_queued
+          msg[1..].freeze
         end
 
         add_round_robin_send_connection(connection)
@@ -71,17 +88,34 @@ module OMQ
       def enqueue(parts)
         raise SocketError, "REQ socket expects send/recv/send/recv order" unless @state == :ready
         @state = :waiting_reply
-        enqueue_round_robin(parts)
+        accepted = false
+        begin
+          @reply_connection = @direct_pipe if @direct_pipe&.direct_recv_queue
+          enqueue_round_robin(parts)
+          accepted = true
+        ensure
+          unless accepted
+            @state            = :ready
+            @reply_connection = nil
+          end
+        end
       end
 
 
       private
 
 
+      # The send pump selects the peer when it takes the request off the queue.
+      def write_batch(connection, batch)
+        @reply_connection = connection
+        super
+      end
+
+
       # REQ prepends empty delimiter frame on the wire.
       #
       def transform_send(parts)
-        parts.dup.unshift(EMPTY_BINARY)
+        parts.dup.unshift(EMPTY_BINARY).freeze
       end
 
     end

data/lib/omq/transport/compressed_connection.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "delegate"
+require_relative "compression_arena"
+
+module OMQ
+  module Transport
+    class CompressedConnection < SimpleDelegator
+      attr_reader :last_wire_size_in
+
+      def initialize(conn, max_message_size:, arena_sentinel:, protocol_error:)
+        super(conn)
+        @max_message_size   = max_message_size
+        @arena_sentinel    = arena_sentinel
+        @protocol_error    = protocol_error
+        @pending_messages  = []
+        @last_wire_size_in = nil
+
+        # The outer framer sees transport envelopes, not logical messages.
+        # Also allow LZ4's worst-case block overhead and dictionary shipment.
+        if max_message_size && conn.respond_to?(:max_message_size=)
+          overhead = max_message_size / 255 + 32 + 20 * (max_message_size / 1_073_741_824 + 1)
+          conn.max_message_size = [max_message_size + overhead, CompressionArena::MAX_WIRE_SIZE].max
+        end
+      end
+
+
+      def receive_message(&block)
+        unless @pending_messages.empty?
+          @last_wire_size_in = 0
+          return @pending_messages.shift
+        end
+
+        loop do
+          parts = __getobj__.receive_message(&block)
+          if parts.any? { |part| part.start_with?(@arena_sentinel) }
+            raise @protocol_error, "arena must be a single-part message" unless parts.size == 1
+
+            @pending_messages = CompressionArena.decode(
+              parts.first, max_message_size: @max_message_size, error: @protocol_error,
+            ) { |data, size| decompress_arena(data, size) }
+            @last_wire_size_in = parts.first.bytesize
+            return @pending_messages.shift
+          end
+
+          decoded = decode_parts(parts)
+          next unless decoded
+
+          @last_wire_size_in = parts.sum(&:bytesize)
+          return decoded
+        end
+      rescue Protocol::ZMTP::Error
+        close
+        raise
+      end
+
+
+      # Fan-out must pass logical messages through the compression layer.
+      def respond_to?(name, include_private = false)
+        return false if [:write_wire, :write_wire_batch].include?(name)
+
+        super
+      end
+    end
+  end
+end

data/lib/omq/transport/compression_arena.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module OMQ
+  module Transport
+    # Shared revision 2 envelope and inner ZMTP framing for LZ4 and Zstd.
+    module CompressionArena
+      HEADER_SIZE   = 16
+      MAX_BYTES     = 65_536
+      MAX_MESSAGES  = 256
+      MAX_PARTS     = 4096
+      MAX_WIRE_SIZE = HEADER_SIZE + MAX_BYTES
+
+      # Account for the decoded String and its entry in the parts array.
+      PART_CHARGE = 64
+
+      module_function
+
+      # Split only messages already admitted by routing. Never wait to fill
+      # an arena, and leave isolated or oversized messages in per-part form.
+      def each_batch(messages, target:)
+        return enum_for(__method__, messages, target: target) unless block_given?
+
+        batch = []
+        bytes = 0
+        parts = 0
+        messages.each do |message|
+          size     = message.sum { |part| part.bytesize + (part.bytesize > 255 ? 9 : 2) }
+          eligible = !message.empty? && size <= target && message.size <= MAX_PARTS
+          if !batch.empty? && (!eligible || bytes + size > target ||
+                              parts + message.size > MAX_PARTS || batch.size == MAX_MESSAGES)
+            yield batch
+            batch = []
+            bytes = 0
+            parts = 0
+          end
+
+          if eligible
+            batch << message
+            bytes += size
+            parts += message.size
+          else
+            yield [message]
+          end
+        end
+        yield batch unless batch.empty?
+      end
+
+
+      def encode(messages, sentinel:, threshold:)
+        data       = messages.map { |parts| Protocol::ZMTP::Codec::Frame.encode_message(parts) }.join.b
+        compressed = yield(data) if data.bytesize >= threshold
+        mode       = compressed && compressed.bytesize < data.bytesize ? 1 : 0
+        sentinel + [mode, "\0\0\0", data.bytesize, messages.size].pack("Ca3VV") + (mode == 1 ? compressed : data)
+      end
+
+
+      def decode(wire, max_message_size:, error:)
+        unless wire.bytesize.between?(HEADER_SIZE, MAX_WIRE_SIZE)
+          raise error, "invalid arena envelope size"
+        end
+
+        mode, reserved, size, count = wire.byteslice(4, 12).unpack("Ca3VV")
+        unless mode <= 1 && reserved == "\0\0\0" &&
+               size.between?(1, MAX_BYTES) && count.between?(1, MAX_MESSAGES)
+          raise error, "invalid arena header"
+        end
+
+        data = wire.byteslice(HEADER_SIZE..)
+        data = yield(data, size) if mode == 1
+        raise error, "arena decoded size mismatch" unless data.bytesize == size
+
+        decode_messages(data, count, max_message_size, error)
+      end
+
+
+      def decode_messages(data, count, max_message_size, error)
+        messages   = []
+        parts      = []
+        offset     = 0
+        part_count = 0
+        spent      = 0
+
+        while offset < data.bytesize
+          flags = data.getbyte(offset)
+          raise error, "invalid arena frame flags" unless (flags & ~3).zero?
+
+          header = (flags & 2).zero? ? 2 : 9
+          raise error, "truncated arena frame header" if offset + header > data.bytesize
+
+          size = header == 2 ? data.getbyte(offset + 1) : data.byteslice(offset + 1, 8).unpack1("Q>")
+          offset += header
+          raise error, "truncated arena frame body" if size > data.bytesize - offset
+
+          part_count += 1
+          raise error, "too many arena parts" if part_count > MAX_PARTS
+
+          spent += size + PART_CHARGE
+          if max_message_size && spent > max_message_size
+            raise error, "decompressed message size exceeds maximum"
+          end
+
+          parts << data.byteslice(offset, size)
+          offset += size
+          next unless (flags & 1).zero?
+
+          messages << parts
+          raise error, "too many arena messages" if messages.size > count
+
+          parts = []
+          spent = 0
+        end
+
+        unless parts.empty? && messages.size == count
+          raise error, "incomplete arena messages"
+        end
+
+        messages
+      end
+    end
+  end
+end

data/lib/omq/transport/inproc/pipe.rb

@@ -62,6 +62,7 @@ module OMQ
           @peer_identity         = peer_identity || "".b
           @peer_socket_type      = peer_type
           @closed                = false
+          @closed_signal         = Async::Promise.new
           @peer                  = nil
           @direct_recv_queue     = nil
           @direct_recv_transform = nil
@@ -88,7 +89,7 @@ module OMQ
 
           return unless @pending_direct
 
-          @pending_direct.each { |msg| queue.enqueue(msg) }
+          @pending_direct.each { |msg| deliver_direct(msg) }
           @pending_direct = nil
         end
 
@@ -115,11 +116,11 @@ module OMQ
           end
 
           if @direct_recv_queue
-            @direct_recv_queue.enqueue(apply_transform(parts))
+            deliver_direct(parts)
           elsif @send_queue
             @send_queue.enqueue(parts)
           else
-            (@pending_direct ||= []) << apply_transform(parts)
+            (@pending_direct ||= []) << parts
           end
         end
 
@@ -205,20 +206,33 @@ module OMQ
         end
 
 
-        # Closes this pipe end and sends a nil sentinel to the peer.
+        # Closes this pipe end and wakes the peer's lifecycle and command reader.
         #
         # @return [void]
         #
         def close
           return if @closed
           @closed = true
+          @closed_signal.resolve(true)
           @send_queue&.enqueue(nil) # close sentinel
         end
 
 
+        # Lets the peer remove this link even when messages bypass recv pumps.
+        def wait_closed
+          @closed_signal.wait
+        end
+
+
         private
 
 
+        def deliver_direct(parts)
+          transformed = apply_transform(parts)
+          @direct_recv_queue.enqueue(transformed) if transformed
+        end
+
+
         def apply_transform(parts)
           if @direct_recv_transform
             @direct_recv_transform.call(parts)

data/lib/omq/transport/inproc.rb

@@ -130,7 +130,7 @@ module OMQ
                   "incompatible socket types: #{client_type} cannot connect to #{server_type}"
           end
 
-          needs_cmds = needs_commands?(client_engine, server_engine, client_type, server_type)
+          needs_cmds = needs_commands?(client_type, server_type)
           client_pipe, server_pipe = make_pipe_pair client_engine, server_engine,
                                                     client_type, server_type, needs_cmds
 
@@ -142,27 +142,12 @@ module OMQ
         # Decides whether a Pipe pair needs command queues.
         # Pipe's fast path skips queues entirely; command queues
         # are only needed for socket types that exchange ZMTP commands
-        # (e.g. ROUTER/DEALER identity, PUB/SUB subscriptions) or when
-        # either side enables QoS ≥ 1.
+        # (e.g. ROUTER/DEALER identity, PUB/SUB subscriptions).
         #
         # @return [Boolean]
         #
-        def needs_commands?(ce, se, ct, st)
-          return true if COMMAND_TYPES.include?(ct) || COMMAND_TYPES.include?(st)
-          return true if qos_enabled?(ce.options) || qos_enabled?(se.options)
-          false
-        end
-
-
-        # QoS integration: core +Options#qos+ defaults to Integer +0+.
-        # When the omq-qos extension is loaded, +#qos+ holds either
-        # +nil+ (QoS 0) or an +OMQ::QoS+ instance (levels 1–3). Treat
-        # both Integer 0 and nil as disabled.
-        def qos_enabled?(options)
-          q = options.qos
-          return false if q.nil?
-          return q != 0 if q.is_a?(Integer)
-          true
+        def needs_commands?(client_type, server_type)
+          COMMAND_TYPES.include?(client_type) || COMMAND_TYPES.include?(server_type)
         end
 
 

data/lib/omq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OMQ
-  VERSION = "0.28.10"
+  VERSION = "0.29.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omq
 version: !ruby/object:Gem::Version
-  version: 0.28.10
+  version: 0.29.0
 platform: ruby
 authors:
 - Patrik Wenger
@@ -16,6 +16,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -23,6 +26,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.5
 - !ruby/object:Gem::Dependency
   name: async
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +120,8 @@ files:
 - lib/omq/scatter_gather.rb
 - lib/omq/single_frame.rb
 - lib/omq/socket.rb
+- lib/omq/transport/compressed_connection.rb
+- lib/omq/transport/compression_arena.rb
 - lib/omq/transport/inproc.rb
 - lib/omq/transport/inproc/pipe.rb
 - lib/omq/transport/ipc.rb
@@ -139,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Pure Ruby ZMQ library
 test_files: []