async-rabbitmq 0.4.0

data/lib/async_rabbitmq/telemetry/open_telemetry.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+# Optional OpenTelemetry tracing. Not loaded by default and not a dependency of
+# the gem: require it yourself, with opentelemetry-api in your bundle.
+#
+#   require "async_rabbitmq/telemetry/open_telemetry"
+#   AsyncRabbitMQ::Telemetry::OpenTelemetry.install
+#
+# The spans and attributes match opentelemetry-instrumentation-bunny, so a
+# service moving over from Bunny keeps the traces and dashboards it had:
+#
+#   * publishing produces a PRODUCER span "<exchange>.<routing key> publish" and
+#     injects the W3C trace context into the message headers,
+#   * basic_get produces a CONSUMER span named for the queue, then renamed to
+#     "<exchange>.<routing key> receive" once the delivery arrives,
+#   * a consumer handler runs inside a CONSUMER span
+#     "<exchange>.<routing key> process", parented by the context extracted from
+#     the message headers, so a trace crosses the broker.
+#
+# One difference from Bunny, forced by the architecture: Bunny creates a receive
+# span for a pushed delivery and parents the process span to it, linking to the
+# producer. Here a pushed delivery goes straight to the handler, so the process
+# span is parented to the producer context directly. Traces join up the same way,
+# with one span per delivery instead of two.
+#
+# Publishing from several fibers on one channel is safe here, and each publish
+# gets its own span: the context is captured when basic_publish is called, not
+# when the frames reach the wire.
+
+require "opentelemetry"
+
+require_relative "../../async_rabbitmq"
+
+module AsyncRabbitMQ
+  module Telemetry
+    module OpenTelemetry
+      TRACER_NAME      = "async-rabbitmq"
+      PROTOCOL         = "AMQP"
+      PROTOCOL_VERSION = "0.9.1"
+      SYSTEM           = "rabbitmq"
+
+      class << self
+        attr_reader :tracer
+
+        # Start tracing. Safe to call more than once: the patch is applied once
+        # and does nothing until a tracer is set.
+        def install(tracer_provider: ::OpenTelemetry.tracer_provider,
+                    tracer_name: TRACER_NAME,
+                    tracer_version: AsyncRabbitMQ::VERSION)
+          @tracer = tracer_provider.tracer(tracer_name, tracer_version)
+          AsyncRabbitMQ::Channel.prepend(ChannelSpans) unless AsyncRabbitMQ::Channel.include?(ChannelSpans)
+          true
+        end
+
+        # Stop tracing. The prepended module stays in the ancestor chain and
+        # falls straight through to the unpatched methods.
+        def uninstall
+          @tracer = nil
+          true
+        end
+
+        def installed?
+          !@tracer.nil?
+        end
+      end
+
+      # Attribute and name construction. Pure functions: everything they need is
+      # passed in, so the publish, get and consume paths cannot drift apart.
+      module Helpers
+        module_function
+
+        # Bunny joins the two with a dot and drops whichever is missing.
+        def destination(exchange, routing_key)
+          [exchange, routing_key].reject { |part| part.nil? || part.to_s.empty? }.join(".")
+        end
+
+        # The default exchange is direct, and a direct exchange addresses one
+        # queue; everything else is a topic in OpenTelemetry's model. The type is
+        # only known for exchanges this client declared.
+        def destination_kind(exchange, exchange_type)
+          return "queue" if exchange.nil? || exchange.to_s.empty?
+          return "queue" if exchange_type.to_s == "direct"
+
+          "topic"
+        end
+
+        def basic_attributes(exchange:, routing_key:, peer_host:, peer_port:, exchange_type: nil)
+          attributes = {
+            "messaging.system"           => SYSTEM,
+            "messaging.destination"      => exchange.to_s,
+            "messaging.destination_kind" => destination_kind(exchange, exchange_type),
+            "messaging.protocol"         => PROTOCOL,
+            "messaging.protocol_version" => PROTOCOL_VERSION,
+            "net.peer.name"              => peer_host,
+            "net.peer.port"              => peer_port
+          }
+          attributes["messaging.rabbitmq.routing_key"] = routing_key if routing_key && !routing_key.to_s.empty?
+          attributes
+        end
+
+        def headers_of(header)
+          return {} unless header.respond_to?(:properties) && header.properties
+
+          header.properties[:headers] || {}
+        end
+
+        def message_id_of(header)
+          return nil unless header.respond_to?(:properties) && header.properties
+
+          header.properties[:message_id]
+        end
+      end
+
+      # Prepended to AsyncRabbitMQ::Channel by .install.
+      module ChannelSpans
+        def basic_publish(payload, exchange: "", routing_key: "", **opts)
+          tracer = OpenTelemetry.tracer
+          return super unless tracer
+
+          tracer.in_span("#{Helpers.destination(exchange, routing_key)} publish",
+                         attributes: otel_attributes(exchange, routing_key), kind: :producer) do
+            # Copy: inject writes traceparent into this hash, and opts[:headers]
+            # is the caller's own object — a frozen constant would raise, and a
+            # hash shared between fibers would race.
+            headers = (opts[:headers] || {}).dup
+            ::OpenTelemetry.propagation.inject(headers)
+            super(payload, exchange: exchange, routing_key: routing_key, **opts.merge(headers: headers))
+          end
+        end
+
+        def basic_publish_batch(payloads, exchange: "", routing_key: "", **opts)
+          tracer = OpenTelemetry.tracer
+          return super unless tracer
+
+          attributes = otel_attributes(exchange, routing_key)
+          attributes["messaging.batch.message_count"] = payloads.is_a?(Array) ? payloads.size : 0
+          tracer.in_span("#{Helpers.destination(exchange, routing_key)} publish",
+                         attributes: attributes, kind: :producer) do
+            headers = (opts[:headers] || {}).dup
+            ::OpenTelemetry.propagation.inject(headers)
+            super(payloads, exchange: exchange, routing_key: routing_key, **opts.merge(headers: headers))
+          end
+        end
+
+        def basic_get(queue_name, manual_ack: true)
+          tracer = OpenTelemetry.tracer
+          return super unless tracer
+
+          attributes = otel_attributes("", nil)
+          attributes["messaging.operation"] = "receive"
+          tracer.in_span("#{queue_name} receive", attributes: attributes, kind: :consumer) do |span|
+            result = super
+            delivery, = result
+            if delivery
+              span["messaging.destination"] = delivery.exchange.to_s
+              span["messaging.destination_kind"] = Helpers.destination_kind(delivery.exchange,
+                                                                            otel_exchange_type(delivery.exchange))
+              span["messaging.rabbitmq.routing_key"] = delivery.routing_key if delivery.routing_key
+              span.name = "#{Helpers.destination(delivery.exchange, delivery.routing_key)} receive"
+            end
+            result
+          end
+        end
+
+        def basic_consume(queue_name, **opts, &block)
+          tracer = OpenTelemetry.tracer
+          return super unless tracer && block
+          # Recovery re-registers consumers with the block it stored, which is
+          # already wrapped; wrapping again would nest a span per reconnect.
+          return super if block.respond_to?(:async_rabbitmq_traced?)
+
+          traced = proc do |delivery, header, body|
+            otel_process_span(tracer, delivery, header) { block.call(delivery, header, body) }
+          end
+          traced.define_singleton_method(:async_rabbitmq_traced?) { true }
+          super(queue_name, **opts, &traced)
+        end
+
+        private
+
+        def otel_attributes(exchange, routing_key)
+          Helpers.basic_attributes(exchange: exchange, routing_key: routing_key,
+                                   peer_host: @session.host, peer_port: @session.port,
+                                   exchange_type: otel_exchange_type(exchange))
+        end
+
+        # Only exchanges this client declared are in the registry; a
+        # pre-existing exchange has no recorded type and is treated as a topic.
+        def otel_exchange_type(exchange)
+          return nil if exchange.nil? || exchange.to_s.empty?
+
+          topology&.exchanges&.find { |recorded| recorded.name == exchange }&.type
+        end
+
+        def otel_process_span(tracer, delivery, header, &block)
+          attributes = otel_attributes(delivery.exchange, delivery.routing_key)
+          attributes["messaging.operation"] = "process"
+          if (message_id = Helpers.message_id_of(header))
+            attributes["messaging.message_id"] = message_id
+          end
+
+          parent = ::OpenTelemetry.propagation.extract(Helpers.headers_of(header))
+          ::OpenTelemetry::Context.with_current(parent) do
+            tracer.in_span("#{Helpers.destination(delivery.exchange, delivery.routing_key)} process",
+                           attributes: attributes, kind: :consumer, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/async_rabbitmq/tls.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module AsyncRabbitMQ
+  # Builds an OpenSSL::SSL::SSLContext from the things people actually have: a
+  # client certificate, its key, and the CA that signed the broker's. Nobody
+  # should need to know about X509::Store to connect over TLS; anyone who needs
+  # more than this passes a ready-made +tls_context:+ to Session instead.
+  #
+  #   AsyncRabbitMQ::Session.new(tls_cert: "client.pem", tls_key: "client.key",
+  #                              tls_ca_certificates: ["ca.pem"])
+  #
+  # Every certificate and key option takes either a file path or PEM text, so
+  # material from a secrets manager works as well as a file. A certificate
+  # value may hold the leaf followed by its chain; a CA value may hold several
+  # certificates; +tls_ca_certificates+ takes one value or a list. With no CAs
+  # given the system trust store is used.
+  module TLS
+    VERSIONS = {
+      "TLS1_2" => OpenSSL::SSL::TLS1_2_VERSION,
+      "TLS1_3" => OpenSSL::SSL::TLS1_3_VERSION
+    }.freeze
+
+    module_function
+
+    def context(cert: nil, key: nil, ca_certificates: nil, verify_peer: true, min_version: :TLS1_2,
+                certificate_store: nil)
+      ctx = OpenSSL::SSL::SSLContext.new
+      # set_params first: it supplies the system trust store, hostname
+      # verification and OpenSSL's recommended options, which we then narrow.
+      ctx.set_params(verify_mode: verify_peer ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE)
+      ctx.verify_hostname = verify_peer
+      ctx.min_version     = version(min_version)
+
+      if cert
+        leaf, *chain = OpenSSL::X509::Certificate.load(material(cert, "tls_cert"))
+        ctx.cert = leaf
+        ctx.extra_chain_cert = chain unless chain.empty?
+      end
+      ctx.key = OpenSSL::PKey.read(material(key, "tls_key")) if key
+      raise ArgumentError, "tls_key given without tls_cert" if key && !cert
+
+      if certificate_store
+        ctx.cert_store = certificate_store
+      elsif ca_certificates
+        ctx.cert_store = store(Array(ca_certificates))
+      end
+      ctx
+    end
+
+    # A trust store holding exactly the given CAs (paths or PEM text, each of
+    # which may contain several certificates).
+    def store(ca_certificates)
+      store = OpenSSL::X509::Store.new
+      ca_certificates.each do |value|
+        OpenSSL::X509::Certificate.load(material(value, "tls_ca_certificates")).each { |c| store.add_cert(c) }
+      end
+      store
+    end
+
+    # PEM text is returned as is; anything else is read as a file path.
+    def material(value, option)
+      text = value.to_s
+      return text if text.include?("-----BEGIN")
+      return File.read(text) if File.file?(text)
+
+      raise ArgumentError, "#{option}: #{value.inspect} is neither PEM text nor a readable file"
+    end
+
+    # :TLS1_2, "TLS1_3", "TLSv1.2", 1.2 or an OpenSSL::SSL::*_VERSION constant.
+    def version(value)
+      return value if value.is_a?(Integer)
+
+      key = value.to_s.sub(/\ATLSv?/i, "TLS").tr(".", "_").upcase
+      key = "TLS#{key}" unless key.start_with?("TLS")
+      VERSIONS.fetch(key) do
+        raise ArgumentError, "tls_min_version #{value.inspect}: expected one of #{VERSIONS.keys.join(', ')}"
+      end
+    end
+  end
+end

data/lib/async_rabbitmq/topology_registry.rb

@@ -0,0 +1,210 @@
+module AsyncRabbitMQ
+  # Records the exchanges, queues and bindings declared through a Session so
+  # they can be re-declared after a reconnect ("topology recovery", as in the
+  # RabbitMQ Java client and Bunny 3.0). Passive declares are not recorded,
+  # deletes and unbinds remove their records, and a server-named queue that
+  # comes back under a new name has its bindings and Queue objects renamed.
+  #
+  # The registry mirrors what the broker will do, not which channel declared
+  # what: a durable queue declared on a channel that was then closed still
+  # exists and is still recovered, and so is an exclusive queue, which lives as
+  # long as the connection. What a channel closing does remove is the
+  # consumers on it, and the broker deletes an auto-delete queue when its last
+  # consumer goes, so consumers are recorded too and an auto-delete queue is
+  # forgotten with its last one, by cancel, broker cancel or channel close.
+  # The bindings go with it, and an auto-delete exchange that just lost its
+  # last binding goes as well, again as the broker does. Consumers on other
+  # connections are not visible here; if one keeps such a queue alive, all
+  # that happens is that recovery does not re-declare a queue that exists.
+  #
+  # An optional +filter+ object may implement any of filter_exchanges(list),
+  # filter_queues(list), filter_queue_bindings(list) and
+  # filter_exchange_bindings(list); each receives the recorded entries and
+  # returns the subset to recover.
+  class TopologyRegistry
+    RecordedExchange = Struct.new(:channel_id, :name, :type, :durable, :auto_delete, :internal, :arguments,
+                                  keyword_init: true)
+    RecordedQueue    = Struct.new(:channel_id, :name, :durable, :exclusive, :auto_delete, :arguments, :server_named,
+                                  :objects, keyword_init: true)
+    RecordedQueueBinding    = Struct.new(:channel_id, :queue, :exchange, :routing_key, :arguments, keyword_init: true)
+    RecordedExchangeBinding = Struct.new(:channel_id, :source, :destination, :routing_key, :arguments, keyword_init: true)
+
+    attr_reader :filter
+
+    def initialize(filter: nil)
+      @filter            = filter
+      @exchanges         = {}   # name => RecordedExchange
+      @queues            = {}   # name => RecordedQueue
+      @queue_bindings    = []
+      @exchange_bindings = []
+      @consumers         = {}   # consumer tag => queue name
+    end
+
+    # --- recording -----------------------------------------------------------
+
+    def record_exchange(channel_id, name, type, durable:, auto_delete:, internal:, arguments:)
+      return if name.empty? || AsyncRabbitMQ::Exchange::PREDEFINED_EXCHANGES.include?(name)
+      @exchanges[name] = RecordedExchange.new(channel_id: channel_id, name: name, type: type.to_s, durable: durable,
+                                              auto_delete: auto_delete, internal: internal, arguments: arguments.dup)
+    end
+
+    def delete_exchange(name)
+      @exchanges.delete(name)
+      @queue_bindings.reject! { |b| b.exchange == name }
+      gone, @exchange_bindings = @exchange_bindings.partition { |b| b.source == name || b.destination == name }
+      # An exchange bound to the deleted one may itself have been auto-delete
+      # and kept alive only by that binding.
+      gone.each { |b| prune_auto_delete_exchange(b.source) if b.destination == name }
+    end
+
+    def record_queue(channel_id, name, durable:, exclusive:, auto_delete:, arguments:, server_named:, object: nil)
+      existing = @queues[name]
+      rec = RecordedQueue.new(channel_id: channel_id, name: name, durable: durable, exclusive: exclusive,
+                              auto_delete: auto_delete, arguments: arguments.dup, server_named: server_named,
+                              objects: existing ? existing.objects : [])
+      rec.objects << object if object && !rec.objects.include?(object)
+      @queues[name] = rec
+    end
+
+    def delete_queue(name)
+      @queues.delete(name)
+      gone, @queue_bindings = @queue_bindings.partition { |b| b.queue == name }
+      gone.each { |b| prune_auto_delete_exchange(b.exchange) }
+    end
+
+    # A server-named queue was re-declared under a different name.
+    def rename_queue(old_name, new_name)
+      rec = @queues.delete(old_name) or return
+      rec.name = new_name
+      @queues[new_name] = rec
+      @queue_bindings.each { |b| b.queue = new_name if b.queue == old_name }
+      @consumers.transform_values! { |queue| queue == old_name ? new_name : queue }
+      rec.objects.each { |q| q.update_name_to(new_name) }
+    end
+
+    def record_queue_binding(channel_id, queue:, exchange:, routing_key:, arguments:)
+      delete_queue_binding(queue: queue, exchange: exchange, routing_key: routing_key, arguments: arguments, prune: false)
+      @queue_bindings << RecordedQueueBinding.new(channel_id: channel_id, queue: queue, exchange: exchange,
+                                                  routing_key: routing_key, arguments: arguments.dup)
+    end
+
+    def delete_queue_binding(queue:, exchange:, routing_key:, arguments:, prune: true)
+      @queue_bindings.reject! do |b|
+        b.queue == queue && b.exchange == exchange && b.routing_key == routing_key && b.arguments == arguments
+      end
+      prune_auto_delete_exchange(exchange) if prune
+    end
+
+    def record_exchange_binding(channel_id, source:, destination:, routing_key:, arguments:)
+      delete_exchange_binding(source: source, destination: destination, routing_key: routing_key,
+                              arguments: arguments, prune: false)
+      @exchange_bindings << RecordedExchangeBinding.new(channel_id: channel_id, source: source, destination: destination,
+                                                        routing_key: routing_key, arguments: arguments.dup)
+    end
+
+    def delete_exchange_binding(source:, destination:, routing_key:, arguments:, prune: true)
+      @exchange_bindings.reject! do |b|
+        b.source == source && b.destination == destination && b.routing_key == routing_key && b.arguments == arguments
+      end
+      prune_auto_delete_exchange(source) if prune
+    end
+
+    # --- consumers -----------------------------------------------------------
+
+    def record_consumer(consumer_tag, queue_name)
+      @consumers[consumer_tag] = queue_name
+    end
+
+    # The consumer is gone: cancelled by the client or the broker, or its
+    # channel closed. If it was the last one on an auto-delete queue, the
+    # broker deletes the queue, so forget the queue and what hung off it.
+    def delete_consumer(consumer_tag)
+      queue_name = @consumers.delete(consumer_tag) or return
+      prune_auto_delete_queue(queue_name)
+    end
+
+    # consumer tag => queue name, for inspection.
+    def consumers
+      @consumers.dup
+    end
+
+    # --- what to recover (filtered) -----------------------------------------
+
+    def exchanges
+      apply(:filter_exchanges, @exchanges.values)
+    end
+
+    def queues
+      apply(:filter_queues, @queues.values)
+    end
+
+    def queue_bindings
+      apply(:filter_queue_bindings, @queue_bindings.dup)
+    end
+
+    def exchange_bindings
+      apply(:filter_exchange_bindings, @exchange_bindings.dup)
+    end
+
+    def empty?
+      @exchanges.empty? && @queues.empty? && @queue_bindings.empty? && @exchange_bindings.empty?
+    end
+
+    # Forget everything. For a connection whose channels have all been
+    # dropped: nothing client-side holds what it declared, so nothing is
+    # re-declared when it reconnects.
+    def clear
+      @exchanges.clear
+      @queues.clear
+      @queue_bindings.clear
+      @exchange_bindings.clear
+      @consumers.clear
+    end
+
+    # Forget only what was tied to the connection that just went: exclusive
+    # and auto-delete queues, server-named queues (whose names are gone with
+    # the connection), auto-delete exchanges, and any binding that referred to
+    # them. Durable topology is kept, so a broker that also lost it — a node
+    # restarted from an empty data directory — still has it re-declared.
+    def clear_transient
+      dropped_queues = @queues.each_value
+                              .select { |q| q.exclusive || q.auto_delete || q.server_named }
+                              .map(&:name)
+      dropped_exchanges = @exchanges.each_value.select(&:auto_delete).map(&:name)
+
+      dropped_queues.each    { |name| @queues.delete(name) }
+      dropped_exchanges.each { |name| @exchanges.delete(name) }
+
+      @queue_bindings.reject! do |b|
+        dropped_queues.include?(b.queue) || dropped_exchanges.include?(b.exchange)
+      end
+      @exchange_bindings.reject! do |b|
+        dropped_exchanges.include?(b.source) || dropped_exchanges.include?(b.destination)
+      end
+      # Consumers belong to the channels that were dropped with the connection.
+      @consumers.clear
+    end
+
+    private
+
+    def prune_auto_delete_queue(name)
+      rec = @queues[name]
+      return unless rec&.auto_delete
+      return if @consumers.value?(name)
+
+      delete_queue(name)
+    end
+
+    def prune_auto_delete_exchange(name)
+      rec = @exchanges[name]
+      return unless rec&.auto_delete
+      return if @queue_bindings.any? { |b| b.exchange == name } || @exchange_bindings.any? { |b| b.source == name }
+
+      delete_exchange(name)
+    end
+
+    def apply(filter_method, list)
+      @filter.respond_to?(filter_method) ? @filter.public_send(filter_method, list) : list
+    end
+  end
+end

data/lib/async_rabbitmq/version.rb

@@ -0,0 +1,3 @@
+module AsyncRabbitMQ
+  VERSION = "0.4.0"
+end

data/lib/async_rabbitmq/versioned_delivery_tag.rb

@@ -0,0 +1,62 @@
+module AsyncRabbitMQ
+  # A broker delivery tag plus the connection generation it was issued on.
+  #
+  # Delivery tags are scoped to a channel on a connection: after a reconnect
+  # the broker starts numbering from 1 again. A handler that was still running
+  # when the connection dropped would otherwise ack a tag from the old
+  # connection against the new channel, where that number now belongs to a
+  # different message (silent loss, or a whole range of them with
+  # multiple: true) or to no message at all (a 406 that closes the channel).
+  #
+  # Channel#basic_ack, #basic_nack and #basic_reject compare the generation
+  # carried here against the channel's current one and drop stale tags.
+  #
+  # It stands in for the Integer it wraps: it converts implicitly (#to_int),
+  # compares and sorts against Integers, and hashes equal to them, so code
+  # that treats a delivery tag as a number keeps working.
+  class VersionedDeliveryTag
+    include Comparable
+
+    attr_reader :tag, :generation
+
+    def initialize(tag, generation)
+      @tag        = tag
+      @generation = generation
+    end
+
+    # True when this tag was issued on an earlier connection than +current+.
+    def stale?(current)
+      @generation < current
+    end
+
+    def to_i
+      @tag
+    end
+    alias to_int to_i
+
+    def coerce(other)
+      [other, @tag]
+    end
+
+    def <=>(other)
+      @tag <=> (other.respond_to?(:to_i) ? other.to_i : other)
+    end
+
+    def ==(other)
+      (@tag <=> (other.respond_to?(:to_i) ? other.to_i : other)) == 0
+    end
+    alias eql? ==
+
+    def hash
+      @tag.hash
+    end
+
+    def to_s
+      @tag.to_s
+    end
+
+    def inspect
+      "#<#{self.class.name} #{@tag} gen=#{@generation}>"
+    end
+  end
+end

data/lib/async_rabbitmq.rb

@@ -0,0 +1,18 @@
+require_relative "async_rabbitmq/version"
+require_relative "async_rabbitmq/log"
+require_relative "async_rabbitmq/errors"
+require_relative "async_rabbitmq/tls"
+require_relative "async_rabbitmq/channel_id_allocator"
+require_relative "async_rabbitmq/notifier"
+require_relative "async_rabbitmq/versioned_delivery_tag"
+require_relative "async_rabbitmq/topology_registry"
+require_relative "async_rabbitmq/frame_io"
+require_relative "async_rabbitmq/queue"
+require_relative "async_rabbitmq/exchange"
+require_relative "async_rabbitmq/channel"
+require_relative "async_rabbitmq/session"
+require_relative "async_rabbitmq/cluster"
+
+module AsyncRabbitMQ
+  # Optional: require "async_rabbitmq/pool" for connection pooling.
+end