async-rabbitmq 0.4.0

data/lib/async_rabbitmq/session.rb

@@ -0,0 +1,1154 @@
+require "async"
+require "async/condition"
+require "async/semaphore"
+require "socket"
+require "openssl"
+require "uri"
+require "amq/uri"
+require_relative "errors"
+require_relative "frame_io"
+require_relative "channel"
+require_relative "sasl"
+
+module AsyncRabbitMQ
+  # Represents one AMQP connection to a RabbitMQ broker.
+  #
+  # Recovery: exponential backoff with configurable interval, max interval,
+  # and retry limit. Re-registers consumers after reconnect.
+  #
+  # Pool interface: implements reusable?, viable?, concurrency, close for Async::Pool.
+  class Session
+    include Instrumented
+
+    CONNECT_TIMEOUT     = 30     # seconds for AMQP handshake
+    RPC_TIMEOUT         = 15     # seconds to wait for a synchronous channel reply (nil = forever)
+    RECOVERY_INITIAL    = 1.0    # seconds
+    RECOVERY_MAX        = 30.0   # seconds
+    RECOVERY_JITTER     = 0.25   # ±25%
+    PROTOCOL_HEADER     = "AMQP\x00\x00\x09\x01".b.freeze
+
+    attr_reader :host, :port, :vhost, :username, :addresses
+
+    # The TopologyRegistry of exchanges, queues and bindings declared through
+    # this session, replayed after a reconnect (see recover_topology:).
+    attr_reader :topology
+
+    # Structured events for metrics and tracing; see Notifier::EVENTS.
+    attr_reader :notifier
+
+    # Build a Session from one or more AMQP URI strings.
+    #
+    #   Session.from_uri("amqp://user:pass@rabbit:5672/myvhost")
+    #   Session.from_uri("amqps://rabbit/myvhost", tls_context: ctx)
+    #   Session.from_uri("amqp://rabbit1:5672/vh", "amqp://rabbit2:5672/vh")
+    #
+    # When multiple URIs are given, credentials, vhost, and TLS settings are
+    # taken from the first URI.  Each URI contributes a host:port pair to the
+    # +addresses+ list used for failover.
+    #
+    # Keyword arguments override anything parsed from the URI(s).
+    #
+    # The standard RabbitMQ URI query parameters are honoured: +heartbeat+,
+    # +connection_timeout+, +channel_max+, +auth_mechanism+ and, for amqps://,
+    # +verify+, +cacertfile+, +certfile+ and +keyfile+ (which build a
+    # +tls_context+ unless one is passed explicitly).
+    def self.from_uri(*uri_strings, **kwargs)
+      new(**options_from_uris(*uri_strings, **kwargs))
+    end
+
+    # The Session.new keyword arguments for one or more URIs; see from_uri.
+    # Cluster.from_uri builds its nodes from the same list.
+    def self.options_from_uris(*uri_strings, **kwargs)
+      raise ArgumentError, "at least one URI string is required" if uri_strings.empty?
+
+      first_opts = uri_options(uri_strings.first)
+      address_list = uri_strings.map do |u|
+        parsed = uri_options(u)
+        "#{parsed[:host] || 'localhost'}:#{parsed[:port]}"
+      end
+
+      # Connection-level settings come from the first URI; addresses from all.
+      opts = first_opts.except(:host, :port)
+      opts[:addresses] = address_list
+      opts.merge(kwargs)
+    end
+
+    # Translate an amqp:// or amqps:// URI into Session.new keyword arguments.
+    #
+    # Parsing is delegated to AMQ::URI from amq-protocol, which percent-decodes
+    # the credentials and vhost (so "pa+ss" stays "pa+ss") and validates the
+    # scheme, the single-segment vhost path and the TLS-only query parameters.
+    def self.uri_options(uri_string)
+      parsed = AMQ::URI.parse(uri_string)
+      query  = ::URI.parse(uri_string).query
+      params = query ? ::URI.decode_www_form(query).to_h : {}
+
+      opts = { tls: !!parsed[:ssl], port: parsed[:port] }
+      opts[:host]     = parsed[:host] if parsed[:host]
+      opts[:username] = parsed[:user] if parsed[:user]
+      opts[:password] = parsed[:pass] if parsed[:pass]
+      # "amqp://host/" yields an empty vhost; treat it as the default "/" rather
+      # than the (rarely intended) vhost literally named "".
+      opts[:vhost] = parsed[:vhost].empty? ? "/" : parsed[:vhost] if parsed.key?(:vhost)
+
+      opts[:heartbeat]       = Integer(params["heartbeat"])          if params.key?("heartbeat")
+      opts[:connect_timeout] = Integer(params["connection_timeout"]) if params.key?("connection_timeout")
+      opts[:channel_max]     = Integer(params["channel_max"])        if params.key?("channel_max")
+      opts[:auth_mechanism]  = params["auth_mechanism"]              if params.key?("auth_mechanism")
+
+      if opts[:tls] && %w[verify cacertfile certfile keyfile].any? { |k| params.key?(k) }
+        opts[:tls_context] = tls_context_from_uri(parsed, params)
+      end
+
+      opts
+    end
+
+    # Build an OpenSSL context from the TLS query parameters of an amqps:// URI,
+    # the same way the tls_cert:/tls_key:/tls_ca_certificates: options do.
+    # Peer verification stays on unless the URI says verify=false explicitly.
+    def self.tls_context_from_uri(parsed, params)
+      verify = params.key?("verify") ? params["verify"] != "false" : true
+      TLS.context(cert: parsed[:certfile], key: parsed[:keyfile], ca_certificates: parsed[:cacertfile],
+                  verify_peer: verify)
+    end
+
+    private_class_method :uri_options, :tls_context_from_uri
+
+    # The canonical [host, port] list from the various input forms.
+    #
+    #   addresses: ["rabbit1:5672", "rabbit2:5673"]    -> [["rabbit1",5672], ["rabbit2",5673]]
+    #   hosts: ["rabbit1", "rabbit2"], port: 5672       -> [["rabbit1",5672], ["rabbit2",5672]]
+    #   host: "rabbit1", port: 5672 (default)           -> [["rabbit1",5672]]
+    def self.address_list(host:, port:, hosts: nil, addresses: nil)
+      if addresses && !addresses.empty?
+        addresses.map do |addr|
+          h, p = addr.to_s.split(":", 2)
+          [h, p ? p.to_i : port]
+        end
+      elsif hosts && !hosts.empty?
+        hosts.map { |h| [h.to_s, port] }
+      else
+        [[host.to_s, port]]
+      end
+    end
+
+    def initialize(
+      host: "localhost",
+      port: 5672,
+      hosts: nil,
+      addresses: nil,
+      hosts_shuffle_strategy: :shuffle,
+      vhost: "/",
+      username: "guest",
+      password: "guest",
+      tls: false,
+      tls_context: nil,
+      tls_cert: nil,
+      tls_key: nil,
+      tls_ca_certificates: nil,
+      verify_peer: true,
+      tls_min_version: :TLS1_2,
+      heartbeat: 60,
+      tcp_user_timeout: nil,
+      frame_max: 131_072,
+      channel_max: 2047,
+      connect_timeout: CONNECT_TIMEOUT,
+      rpc_timeout: RPC_TIMEOUT,
+      auth_mechanism: nil,
+      connection_name: nil,
+      auto_recover: true,
+      recovery_attempts: nil,
+      recovery_interval: RECOVERY_INITIAL,
+      recovery_max_interval: RECOVERY_MAX,
+      recover_topology: true,
+      topology_recovery_filter: nil,
+      instrumenter: nil,
+      notifier: nil,
+      logger: Log.new
+    )
+      @addresses = self.class.address_list(host: host, port: port, hosts: hosts, addresses: addresses)
+      @host                 = @addresses.first[0]
+      @port                 = @addresses.first[1]
+      @hosts_shuffle_strategy = hosts_shuffle_strategy
+      @vhost                = vhost
+      @username             = username
+      @password             = password
+      # Certificate material or a ready context means TLS, whatever tls: says.
+      @tls                  = tls || !(tls_context || tls_cert || tls_key || tls_ca_certificates).nil?
+      @tls_context          = tls_context
+      @tls_cert             = tls_cert
+      @tls_key              = tls_key
+      @tls_ca_certificates  = tls_ca_certificates
+      @verify_peer          = verify_peer
+      @tls_min_version      = tls_min_version
+      @heartbeat            = heartbeat
+      @tcp_user_timeout     = tcp_user_timeout
+      @frame_max            = frame_max
+      @channel_max          = channel_max
+      @connect_timeout      = connect_timeout
+      @rpc_timeout          = rpc_timeout
+      @auth_mechanism       = auth_mechanism
+      @connection_name      = connection_name
+      @auto_recover         = auto_recover
+      @recovery_attempts    = recovery_attempts    # nil = unlimited
+      @recovery_interval    = recovery_interval
+      @recovery_max_interval = recovery_max_interval
+      @recover_topology     = recover_topology
+      @topology             = TopologyRegistry.new(filter: topology_recovery_filter)
+      @logger               = logger
+      @notifier             = notifier || Notifier.new(logger: logger)
+      @notifier.subscribe { |name, payload| instrumenter.call(name, payload) } if instrumenter
+
+      @state             = :closed
+      @frame_io          = nil
+      @channels          = {}       # channel_id => Channel; a closed channel's id is free again
+      @channel_mutex     = Async::Semaphore.new(1)
+      @negotiated_hb     = nil
+      @negotiated_fm     = nil
+      @negotiated_cmax   = nil
+      @recovery_in_progress = false
+      @recovery_interrupted = false
+      @closed_by_user    = false
+      @connecting        = false
+      @open_condition    = nil
+      @last_frame_at     = nil
+      @heartbeat_task    = nil
+      @channel0_task     = nil    # drains channel-0 queue; handles connection.blocked/unblocked
+      @recovery_task     = nil
+      @recovery_wakeup   = nil    # Async::Condition to interrupt the retry sleep
+      @session_root_task = nil    # top-level task; recovery is spawned here so it
+                                  # survives reader-task Cancel propagation
+      @on_blocked              = nil
+      @on_unblocked            = nil
+      @update_secret_condition = nil
+      @on_recovery_attempt     = nil
+      @on_recovery             = nil
+      @on_recovery_exhausted   = nil
+      @on_connection_lost      = nil
+    end
+
+    # Connect and complete AMQP handshake. Raises ConnectionTimeoutError if
+    # the handshake does not complete within +connect_timeout+ seconds.
+    # Raises NotOpenError if already connected.
+    def connect
+      raise NotOpenError, "Session is already connected" if open?
+      @connecting        = true
+      @closed_by_user    = false   # a previous failed connect must not disable recovery
+      @session_root_task = Async::Task.current
+
+      last_error = nil
+      started_at = instrument_clock
+      shuffled_addresses.each do |target_host, target_port|
+        begin
+          Async::Task.current.with_timeout(@connect_timeout) do
+            raw_socket = open_socket(target_host, target_port)
+            @frame_io  = build_frame_io(raw_socket)
+            @frame_io.start(spawn: method(:spawn_background))
+            handshake
+            start_heartbeat_task
+            start_channel0_monitor_task
+            @host  = target_host
+            @port  = target_port
+            @state = :open
+            @channel_ids = fresh_channel_ids
+          end
+          @connecting = false
+          instrument("connection.open") do
+            { host: @host, port: @port, vhost: @vhost, tls: @tls, heartbeat: @negotiated_hb,
+              frame_max: @negotiated_fm, channel_max: @negotiated_cmax,
+              duration: started_at ? instrument_elapsed(started_at) : nil }
+          end
+          return
+        rescue Async::TimeoutError => e
+          @frame_io&.stop rescue nil
+          @frame_io = nil
+          last_error = e
+        rescue SystemCallError, EOFError, IOError, SocketError => e
+          # TCP connect failed (refused, unreachable, timed out) or the peer
+          # dropped the connection mid-handshake (reset, EOF, shutdown).
+          @frame_io&.stop rescue nil
+          @frame_io = nil
+          last_error = e
+        rescue OpenSSL::SSL::SSLError => e
+          @frame_io&.stop rescue nil
+          @frame_io = nil
+          last_error = e
+        rescue AuthenticationError
+          # Credentials or vhost access refused: other nodes will say the same.
+          cleanup_after_failed_connect
+          raise
+        rescue ConnectionError, ChannelError => e
+          # The broker refused the handshake (e.g. 530 NOT_ALLOWED for a missing
+          # vhost, 320 CONNECTION_FORCED from a node in maintenance). Stop the
+          # reader/writer tasks on this socket and try the next address.
+          @frame_io&.stop rescue nil
+          @frame_io = nil
+          last_error = e
+        end
+      end
+
+      # All addresses exhausted
+      cleanup_after_failed_connect
+      tried = @addresses.map { |h, p| "#{h}:#{p}" }.join(", ")
+      case last_error
+      when Async::TimeoutError
+        raise ConnectionTimeoutError, "AMQP handshake did not complete within #{@connect_timeout}s (tried #{tried})"
+      when OpenSSL::SSL::SSLError
+        raise ConnectionTimeoutError, "TLS handshake failed (tried #{tried}) — #{last_error.message}"
+      when ConnectionError, ChannelError
+        raise last_error
+      else
+        raise ConnectionTimeoutError, "Could not connect to any host (tried #{tried}) — #{last_error&.message}"
+      end
+    end
+
+    def open?
+      @state == :open
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    # Internal. Run a long-lived background loop (channel dispatch, heartbeat,
+    # channel-0 monitor).
+    #
+    # These must not become children of whichever task happened to open the
+    # channel or the session: a short-lived caller that finishes, or is stopped,
+    # would silently take its dispatch loop with it while the channel still
+    # reported itself open. Parent them at the reactor instead, and mark them
+    # transient so they never hold the reactor open by themselves — #close
+    # stops them explicitly.
+    #
+    # @api private
+    def spawn_background(&block)
+      # Looked up per call, never memoised: a Session reused across two
+      # separate Sync/Async reactors would otherwise keep spawning onto the
+      # first, finished one.
+      Async::Task.current.root.async(transient: true, &block)
+    end
+
+    # Close the session gracefully. Also works if recovery is in progress.
+    def close
+      return if closed?
+      @closed_by_user = true
+      @recovery_wakeup&.signal rescue nil
+      # Unblock any fibers waiting on channel replies (e.g. wait_for inside
+      # reopen_after_recovery) so that @recovery_task.cancel below can actually
+      # terminate the task rather than leaving it stuck on an Async::Condition.
+      conn_error = ConnectionError.new(code: 0, text: "Session closed by user")
+      @channels.values.each { |ch| ch.mark_closed!(conn_error) rescue nil }
+      if open?
+        @state = :closing
+        @channel0_task&.cancel rescue nil
+        send_connection_close rescue nil
+      end
+      @heartbeat_task&.cancel rescue nil
+      @channel0_task&.cancel rescue nil
+      @recovery_task&.cancel rescue nil
+      @frame_io&.stop rescue nil
+      @state = :closed
+      instrument("connection.closed") { { reason: :user, host: @host, port: @port } }
+    end
+
+    # Open a new channel. Returns an AsyncRabbitMQ::Channel.
+    # +pool_size+ bounds concurrent consumer-handler fibers on the channel
+    # (Bunny-parity: default 1). basic_qos will auto-adjust it to prefetch_count.
+    def open_channel(pool_size: 1)
+      raise NotOpenError, "Session is not open" unless open?
+      channel = @channel_mutex.acquire do
+        channel_id = next_channel_id
+        Channel.new(channel_id, self, @frame_io, frame_max: @negotiated_fm || @frame_max, logger: @logger,
+                    pool_size: pool_size, rpc_timeout: @rpc_timeout).tap { |ch| @channels[channel_id] = ch }
+      end
+      channel.open
+      channel
+    end
+
+    # Open a channel, yield it to the block, and ensure it is closed afterward.
+    def with_channel(pool_size: 1)
+      ch = open_channel(pool_size: pool_size)
+      begin
+        yield ch
+      ensure
+        ch.close rescue nil
+      end
+    end
+
+    # Rotate the secret this connection authenticated with, without
+    # reconnecting (connection.update-secret, RabbitMQ 3.8+). Used with the
+    # OAuth 2 auth backend to hand the broker a refreshed access token before
+    # the current one expires. The new secret is also used for reconnects.
+    # Raises ConnectionError if the broker refuses (e.g. an auth backend that
+    # does not support secret updates closes the connection).
+    def update_secret(new_secret, reason = "secret update")
+      raise NotOpenError, "Session is not open" unless open?
+
+      cond = @update_secret_condition = Async::Condition.new
+      @frame_io.write_frame(AMQ::Protocol::Connection::UpdateSecret.encode(new_secret, reason).encode)
+      result = Async::Task.current.with_timeout(@rpc_timeout || RPC_TIMEOUT) { cond.wait }
+      raise result if result.is_a?(Exception)
+
+      @password = new_secret
+      true
+    rescue Async::TimeoutError
+      raise RpcTimeoutError, "No reply to connection.update-secret within #{@rpc_timeout || RPC_TIMEOUT}s"
+    ensure
+      @update_secret_condition = nil
+    end
+
+    # Check whether a queue exists on the broker without creating it.
+    # Opens a temporary channel and performs a passive declare.
+    def queue_exists?(name)
+      with_channel { |ch| ch.queue(name, passive: true) }
+      true
+    rescue ChannelError
+      false
+    end
+
+    # Check whether an exchange exists on the broker without creating it.
+    # Opens a temporary channel and performs a passive declare.
+    def exchange_exists?(name)
+      with_channel { |ch| ch.exchange(name, passive: true) }
+      true
+    rescue ChannelError
+      false
+    end
+
+    # Register a callback invoked when the broker sends connection.blocked.
+    # The block receives the reason string from the broker.
+    def on_blocked(&block)
+      @on_blocked = block
+    end
+
+    # Register a callback invoked when the broker sends connection.unblocked.
+    def on_unblocked(&block)
+      @on_unblocked = block
+    end
+
+    # Register a callback invoked at the start of each recovery attempt.
+    # The block receives the attempt number (1-based).
+    def on_recovery_attempt(&block)
+      @on_recovery_attempt = block
+    end
+
+    # Register a callback invoked after recovery succeeds.
+    # The block receives the session.
+    def on_recovery(&block)
+      @on_recovery = block
+    end
+
+    # Subscribe to structured events from this session and its channels. The
+    # block receives (name, payload). +pattern+ is nil for every event, a
+    # String for one name or a prefix ending in a dot, or a Regexp. Returns a
+    # handle for notifier.unsubscribe. See Notifier::EVENTS for the taxonomy.
+    #
+    #   session.on_event("message.") { |name, payload| statsd.increment(name) }
+    def on_event(pattern = nil, &block)
+      @notifier.subscribe(pattern, &block)
+    end
+
+    # Register a callback invoked when recovery attempts are exhausted.
+    # Only fires when recovery_attempts is set to a finite number.
+    # The block receives the session.
+    def on_recovery_exhausted(&block)
+      @on_recovery_exhausted = block
+    end
+
+    # Register a callback invoked when the connection is lost, before
+    # recovery starts (with auto_recover: false, after the channels have
+    # been closed). The block receives the session and the error.
+    def on_connection_lost(&block)
+      @on_connection_lost = block
+    end
+
+    # The channels open or recovering on this connection.
+    def channels
+      @channels.values
+    end
+
+    def channel_count
+      @channels.size
+    end
+
+    # Record a rotated secret for the next connect or reconnect without
+    # sending it: #update_secret does this itself on a live connection.
+    # For a session that is down while the secret rotates.
+    def store_secret(new_secret)
+      @password = new_secret
+    end
+
+    # Called by Channel when it closes itself.
+    def channel_closed(channel_id)
+      @channel_mutex.acquire do
+        @channels.delete(channel_id)
+        @channel_ids&.release(channel_id)
+      end
+      @frame_io&.unregister_channel(channel_id)
+    end
+
+    # Put a channel the broker closed back into the channel table and reopen
+    # it on the current connection under its original id (Channel#reopen).
+    def reopen_channel(channel, state: :open)
+      raise NotOpenError, "Session is not open" unless open?
+      @channel_mutex.acquire do
+        existing = @channels[channel.channel_id]
+        if existing && !existing.equal?(channel)
+          raise Error, "Channel id #{channel.channel_id} is in use by another channel"
+        end
+        @channel_ids ||= fresh_channel_ids
+        @channel_ids.reserve(channel.channel_id) unless existing
+        @channels[channel.channel_id] = channel
+      end
+      channel.reopen_on(@frame_io, state: state)
+    end
+
+    # Called by a channel when topology recovery re-declared a server-named
+    # queue under a new name: update the registry and every consumer of it.
+    def queue_renamed(old_name, new_name)
+      @logger.info("Server-named queue #{old_name} recovered as #{new_name}")
+      @topology.rename_queue(old_name, new_name)
+      @channels.values.each { |ch| ch.rename_consumer_queue(old_name, new_name) }
+    end
+
+    # --- Async::Pool resource interface ---
+
+    # Can this session be returned to the pool?
+    def reusable?
+      open? && !@recovery_in_progress
+    end
+
+    # Is the underlying socket alive?
+    def viable?
+      open?
+    end
+
+    # Max concurrent channels (negotiated with broker, or AMQP max 2047).
+    def concurrency
+      @negotiated_cmax || 2047
+    end
+
+    # --- Internal ---
+
+    # Called by FrameIO when a connection-level error triggers recovery.
+    def trigger_recovery(error, from: nil)
+      return if @closed_by_user
+      # A frame_io that a later connection attempt has already replaced is
+      # still draining its dead socket, and its errors are not the live
+      # connection's. Letting them through made a reconnect look like it had
+      # dropped again the moment it succeeded.
+      return if from && !@frame_io.equal?(from)
+
+      if @connecting
+        # The handshake is in flight in another fiber; hand it the IO error so
+        # it fails now (as a connect failure) rather than sitting in
+        # wait_channel0_method until the connect timeout expires.
+        q0 = @frame_io&.channel_queue(0)
+        q0&.push([:method, error]) rescue nil
+        return
+      end
+
+      instrument("connection.lost") do
+        { host: @host, port: @port, error: error.class.name, message: error.message,
+          recovering: @auto_recover }
+      end
+
+      unless @auto_recover
+        @state = :closed
+        conn_error = ConnectionError.new(code: 0, text: "Connection lost: #{error.message}")
+        @channels.values.each { |ch| ch.mark_closed!(conn_error) rescue nil }
+        connection_lost(error)
+        @frame_io&.stop rescue nil
+        return
+      end
+
+      @logger.warn("Connection lost (#{error.class}: #{error.message}). Starting recovery...")
+
+      if @recovery_in_progress
+        # Connection died again while a recovery attempt is in progress.
+        # Unblock any fiber stuck in wait_channel0_method or channel wait_for
+        # so that the current recover_loop iteration fails fast and retries.
+        recovery_error = ConnectionError.new(code: 0, text: "Connection lost during recovery")
+        @recovery_interrupted = true
+        q0 = @frame_io&.channel_queue(0)
+        q0&.push([:method, recovery_error]) rescue nil
+        @channels.each_value { |ch| ch.interrupt_wait!(recovery_error) rescue nil }
+        return
+      end
+
+      @recovery_in_progress = true
+      @state = :recovering
+
+      # Stop the channel-0 monitor so it doesn't race on the stale queue.
+      # Also unblock any fibers stuck in write_frame waiting on connection.blocked.
+      @channel0_task&.cancel rescue nil
+      @channel0_task = nil
+      @frame_io&.set_unblocked rescue nil
+
+      # Waits in flight raise ConnectionError instead of hanging; new operations
+      # on the channels park until they are reopened after reconnect.
+      conn_error = ConnectionError.new(code: 0, text: "Connection lost: #{error.message}")
+      @channels.values.each { |ch| ch.mark_recovering!(conn_error) rescue nil }
+      @update_secret_condition&.signal(conn_error)
+      # Tell the owner now, while the channels are still in the table: a
+      # Cluster in :drop mode gives them up here, before they are reopened.
+      connection_lost(error)
+
+      # Schedule recover_loop BEFORE stopping old frame_io. old_io.stop
+      # cancels reader/writer tasks; if we ARE the reader task, cancel raises
+      # Async::Cancel (< Exception) which bypasses `rescue nil` and propagates,
+      # so anything after stop might not run.
+      # Parent recovery at the reactor, not at Async::Task.current — which here
+      # is usually the reader task, whose Cancel would propagate into a child
+      # recovery task — and not at the task that called connect either, since
+      # that one may be short-lived while the session outlives it.
+      @recovery_task = spawn_background { recover_loop }
+
+      # Stop the old frame_io: closes the dead socket, pushes nil to channel
+      # queues (unblocking wait_channel0_method), and cancels writer task.
+      # Reader task cancel may raise Async::Cancel — recovery is already scheduled.
+      old_io    = @frame_io
+      @frame_io = nil
+      old_io&.stop rescue nil
+    end
+
+    def frame_max
+      @negotiated_fm || @frame_max
+    end
+
+    private
+
+    # Hand the connection-lost callback its error without letting a failure
+    # in it stop recovery.
+    def connection_lost(error)
+      @on_connection_lost&.call(self, error)
+    rescue => e
+      @logger.error("on_connection_lost callback failed: #{e.class}: #{e.message}")
+    end
+
+    # Return the address list in the order they should be tried for this
+    # connect/recovery cycle.
+    def shuffled_addresses
+      case @hosts_shuffle_strategy
+      when :shuffle then @addresses.shuffle
+      when :none    then @addresses.dup
+      when Proc     then @hosts_shuffle_strategy.call(@addresses)
+      else               @addresses.shuffle
+      end
+    end
+
+    # Stop any in-flight frame_io / recovery tasks spawned during a failed connect.
+    def cleanup_after_failed_connect
+      @closed_by_user = true
+      @connecting     = false
+      @recovery_task&.cancel rescue nil
+      @recovery_task = nil
+      @heartbeat_task&.cancel rescue nil
+      @channel0_task&.cancel rescue nil
+      @frame_io&.stop rescue nil
+      @frame_io = nil
+      @state = :closed
+    end
+
+    def open_socket(target_host = @host, target_port = @port)
+      if @tls
+        require "openssl"
+        ctx = @tls_context || build_tls_context
+        raw = TCPSocket.new(target_host, target_port)
+        # SSLSocket does not close the socket it wraps when the handshake fails,
+        # and a failed connect has no frame_io to stop, so without this a broker
+        # with a bad certificate leaks one descriptor per recovery attempt.
+        handshaked = false
+        begin
+          ssl = OpenSSL::SSL::SSLSocket.new(raw, ctx)
+          # Without this, closing the SSL socket sends close_notify but leaves
+          # the TCP socket open until the GC finalises it, so every amqps
+          # reconnect strands a connection (Amazon MQ, for example).
+          ssl.sync_close = true
+          ssl.hostname = target_host
+          ssl.connect
+          handshaked = true
+          apply_socket_timeouts(raw)
+          ssl
+        ensure
+          raw.close unless handshaked
+        end
+      else
+        TCPSocket.new(target_host, target_port).tap { |sock| apply_socket_timeouts(sock) }
+      end
+    end
+
+    # Bound how long the kernel will retransmit unacknowledged data before it
+    # gives up on the socket. Without it a peer that disappears mid-write (a
+    # network partition with a full send buffer) is only noticed when the TCP
+    # retransmit timer expires, which on Linux is around 15 minutes — far
+    # longer than the heartbeat timeout we promise callers.
+    #
+    # TCP_USER_TIMEOUT is Linux-only; elsewhere the heartbeat task's own write
+    # timeout is the backstop, so a missing constant is not an error.
+    def apply_socket_timeouts(sock)
+      millis = @tcp_user_timeout
+      millis ||= (@heartbeat.to_i > 0 ? @heartbeat.to_i * 2 * 1000 : nil)
+      return unless millis && millis > 0
+      return unless Socket.const_defined?(:TCP_USER_TIMEOUT)
+
+      sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_USER_TIMEOUT, millis)
+    rescue StandardError => e
+      # Unsupported on this platform or socket type; the heartbeat still covers us.
+      @logger.debug("Could not set TCP_USER_TIMEOUT: #{e.class}: #{e.message}")
+    end
+
+    def build_tls_context
+      TLS.context(cert: @tls_cert, key: @tls_key, ca_certificates: @tls_ca_certificates,
+                  verify_peer: @verify_peer, min_version: @tls_min_version)
+    end
+
+    def build_frame_io(raw_socket)
+      io = FrameIO.new(raw_socket, logger: @logger)
+      # Register channel 0 for connection-level frames
+      io.register_channel(0)
+
+      # Override trigger_recovery to delegate to Session
+      session = self
+      io.define_singleton_method(:trigger_recovery) { |error| session.trigger_recovery(error, from: io) }
+
+      # Update heartbeat timestamp on every received frame
+      io.on_frame = -> { @last_frame_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      io
+    end
+
+    def handshake
+      # Send AMQP protocol header directly on the socket
+      @frame_io.instance_variable_get(:@socket).write(PROTOCOL_HEADER)
+
+      # connection.start — negotiate SASL mechanism
+      start = wait_channel0_method(AMQ::Protocol::Connection::Start)
+      sasl = SASL.negotiate(
+        start.mechanisms,
+        preferred: @auth_mechanism,
+        username: @username,
+        password: @password
+      )
+      send_connection_start_ok(sasl)
+
+      # connection.tune (broker may send connection.secure challenges first)
+      msg = wait_channel0_method(AMQ::Protocol::Connection::Tune, AMQ::Protocol::Connection::Secure)
+      while msg.is_a?(AMQ::Protocol::Connection::Secure)
+        @logger.debug("Received connection.secure SASL challenge for #{sasl.mechanism_name}")
+        @frame_io.write_frame(AMQ::Protocol::Connection::SecureOk.encode(sasl.challenge_response(msg.challenge)).encode)
+        msg = wait_channel0_method(AMQ::Protocol::Connection::Tune, AMQ::Protocol::Connection::Secure)
+      end
+      @negotiated_hb   = negotiate_heartbeat(msg.heartbeat)
+      @negotiated_fm   = negotiate_frame_max(msg.frame_max)
+      @negotiated_cmax = negotiate_channel_max(msg.channel_max)
+      send_connection_tune_ok
+
+      # connection.open
+      send_connection_open
+      wait_channel0_method(AMQ::Protocol::Connection::OpenOk)
+    end
+
+    def wait_channel0_method(*expected_classes)
+      queue = @frame_io.channel_queue(0)
+      loop do
+        msg = queue.pop
+        # nil sentinel — pushed by frame_io.stop or interrupt during recovery
+        raise ConnectionError, "Connection closed while waiting for #{expected_classes.join(', ')}" if msg.nil?
+        next unless msg[0] == :method
+        method = msg[1]
+        # An exception pushed directly by trigger_recovery to unblock this wait
+        # (ConnectionError during recovery, or the raw IO error during connect).
+        raise method if method.is_a?(Exception)
+        if expected_classes.any? { |c| method.is_a?(c) }
+          return method
+        elsif method.is_a?(AMQ::Protocol::Connection::Close)
+          code = method.reply_code
+          text = method.reply_text
+          # connection.close is always connection-level. 403 ACCESS_REFUSED here
+          # means the credentials or the vhost access were rejected.
+          if code == 403
+            raise AuthenticationError.new(code: code, text: text)
+          else
+            raise ConnectionError.new(code: code, text: text)
+          end
+        end
+      end
+    end
+
+    def send_connection_start_ok(sasl)
+      props = {
+        "product"      => "async-rabbitmq",
+        "version"      => AsyncRabbitMQ::VERSION,
+        "platform"     => "Ruby #{RUBY_VERSION}",
+        "information"  => "https://github.com/womblep/async-rabbitmq",
+        # Extensions this client understands. authentication_failure_close makes
+        # RabbitMQ answer bad credentials with connection.close 403 instead of
+        # silently dropping the TCP connection (https://www.rabbitmq.com/docs/auth-notification).
+        "capabilities" => {
+          "publisher_confirms"           => true,
+          "consumer_cancel_notify"       => true,
+          "exchange_exchange_bindings"   => true,
+          "basic.nack"                   => true,
+          "connection.blocked"           => true,
+          "authentication_failure_close" => true,
+        },
+      }
+      props["connection_name"] = @connection_name if @connection_name
+      @frame_io.write_frame(
+        AMQ::Protocol::Connection::StartOk.encode(
+          props,
+          sasl.mechanism_name,
+          sasl.initial_response,
+          "en_US"
+        ).encode
+      )
+    end
+
+    def send_connection_tune_ok
+      @frame_io.write_frame(
+        AMQ::Protocol::Connection::TuneOk.encode(
+          @negotiated_cmax,
+          @negotiated_fm,
+          @negotiated_hb
+        ).encode
+      )
+    end
+
+    def send_connection_open
+      @frame_io.write_frame(
+        AMQ::Protocol::Connection::Open.encode(@vhost).encode
+      )
+    end
+
+    def send_connection_close
+      # Bound the whole handshake, write included: the broker may never answer,
+      # and the write queue may be full behind a stalled socket.
+      Async::Task.current.with_timeout(5) do
+        @frame_io.write_frame(
+          AMQ::Protocol::Connection::Close.encode(200, "Goodbye", 0, 0).encode
+        )
+        wait_channel0_method(AMQ::Protocol::Connection::CloseOk)
+      end
+    rescue Async::TimeoutError, ConnectionError, ChannelError, IOError => e
+      # Broker didn't answer in time — proceed with the forced close below, which
+      # takes the rest of the write queue with it. basic_publish only queues
+      # frames, so without confirms this is the publisher's only hint.
+      unwritten = @frame_io&.pending_writes.to_i
+      return unless unwritten.positive?
+
+      @logger.warn("Close discarded #{unwritten} queued write(s) (#{e.class})")
+    end
+
+    def negotiate_heartbeat(broker_hb)
+      return @heartbeat if broker_hb == 0
+      return broker_hb  if @heartbeat == 0
+      [@heartbeat, broker_hb].min
+    end
+
+    def negotiate_frame_max(broker_fm)
+      return @frame_max if broker_fm == 0
+      [@frame_max, broker_fm].min
+    end
+
+    # 0 means "no limit" on either side; otherwise the lower value wins.
+    def negotiate_channel_max(broker_cmax)
+      return (@channel_max == 0 ? 2047 : @channel_max) if broker_cmax == 0
+      return broker_cmax if @channel_max == 0
+      [@channel_max, broker_cmax].min
+    end
+
+    # The negotiated value T is the heartbeat *timeout*. RabbitMQ and the
+    # reference clients send a heartbeat every T/2 and treat the peer as dead
+    # after roughly two missed heartbeats, so we send at T/2 as well and
+    # declare the broker dead when nothing has arrived for 2×T.
+    def start_heartbeat_task
+      timeout = @negotiated_hb || 60
+      return if timeout == 0
+      interval   = timeout / 2.0
+      dead_after = timeout * 2
+      # Seed the timestamp now; the on_frame callback will keep it fresh.
+      @last_frame_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      @heartbeat_task = spawn_background do
+        loop do
+          sleep interval
+          break unless open?
+
+          # Liveness first. The write below needs the socket lock, and after a
+          # partition the writer can be parked inside @socket.write holding it
+          # with a full send buffer. Checking afterwards meant this task blocked
+          # with the rest and nothing declared the peer dead until the kernel
+          # gave up retransmitting — around 15 minutes on Linux.
+          last = @last_frame_at
+          if last
+            elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - last
+            if elapsed > dead_after
+              trigger_recovery(HeartbeatTimeoutError.new("No frame received in #{elapsed.round(1)}s (timeout #{dead_after}s)"))
+              break
+            end
+          end
+
+          # Bounded so a socket that never drains cannot park this task either.
+          # Bounded by the full dead-peer window, not by the send interval: the
+          # heartbeat needs the socket lock, and a large batch going out over a
+          # slow link can legitimately hold it for longer than one interval.
+          begin
+            Async::Task.current.with_timeout(dead_after) { @frame_io.write_heartbeat }
+            instrument("heartbeat.sent") { { interval: interval } }
+          rescue Async::TimeoutError
+            trigger_recovery(HeartbeatTimeoutError.new("Heartbeat write blocked for #{dead_after}s — peer is not reading"))
+            break
+          end
+        end
+      end
+    end
+
+    # The lowest channel id not in use, so ids come back when channels close.
+    # A bare counter ran out at 65535 (the channel field is 16 bits; 65536
+    # encodes as channel 0 and the broker drops the connection) and let the
+    # process open more channels than channel_max, which the broker also
+    # answers by closing the connection. Called with @channel_mutex held.
+    def next_channel_id
+      @channel_ids ||= fresh_channel_ids
+      @channel_ids.allocate or
+        raise ChannelLimitError, "channel_max #{@channel_ids.limit} reached: #{@channels.size} channels open on this connection"
+    end
+
+    # The negotiated channel_max; 0 means "no limit", which the protocol caps
+    # at 65535 through the width of the frame header's channel field.
+    def channel_limit
+      max = @negotiated_cmax || @channel_max
+      max.nil? || max.zero? ? 65_535 : max
+    end
+
+    # A new allocator sized by the current negotiation, with the ids of the
+    # channels we already have marked as taken. Built lazily and again after
+    # each handshake, since a reconnect may negotiate a different channel_max
+    # while the channels keep their ids.
+    def fresh_channel_ids
+      ChannelIdAllocator.new(channel_limit).tap do |ids|
+        @channels.each_key { |id| ids.reserve(id) }
+      end
+    end
+
+    def recover_loop
+      delay = @recovery_interval
+      attempts = 0
+      recovery_started_at = instrument_clock
+
+      loop do
+        break if @closed_by_user
+
+        attempts += 1
+
+        # Check retry limit (nil = unlimited)
+        if @recovery_attempts && attempts > @recovery_attempts
+          @logger.warn("Recovery exhausted after #{@recovery_attempts} attempt(s)")
+          @state = :closed
+          @recovery_in_progress = false
+          exhausted = ConnectionError.new(code: 0, text: "Recovery exhausted after #{@recovery_attempts} attempt(s)")
+          @channels.values.each { |ch| ch.mark_closed!(exhausted) rescue nil }
+          instrument("recovery.exhausted") { { attempts: @recovery_attempts, reason: :attempts_exceeded } }
+          @on_recovery_exhausted&.call(self)
+          return
+        end
+
+        jitter    = delay * RECOVERY_JITTER * (rand * 2 - 1)
+        wait_secs = delay + jitter
+
+        # Sleep until the delay expires OR session.close signals @recovery_wakeup.
+        @recovery_wakeup = Async::Condition.new
+        Async::Task.current.with_timeout(wait_secs) do
+          @recovery_wakeup.wait
+        end rescue nil   # TimeoutError is normal; Condition#signal raises nothing
+        @recovery_wakeup = nil
+
+        break if @closed_by_user
+
+        @logger.info("Recovery attempt #{attempts} (delay was #{delay.round(1)}s)...")
+        instrument("recovery.attempt") { { attempt: attempts, delay: wait_secs } }
+        @on_recovery_attempt&.call(attempts)
+
+        connected = false
+        shuffled_addresses.each do |target_host, target_port|
+          break if @closed_by_user
+          begin
+            # Wrap the entire reconnect in a timeout so a dead socket doesn't hang forever.
+            Async::Task.current.with_timeout(@connect_timeout) do
+              raw_socket = open_socket(target_host, target_port)
+              @frame_io  = build_frame_io(raw_socket)
+              @frame_io.start(spawn: method(:spawn_background))
+              handshake
+            end
+            @host = target_host
+            @port = target_port
+            connected = true
+            break
+          rescue AuthenticationError => e
+            # The credentials no longer work; retrying cannot help.
+            @frame_io&.stop rescue nil
+            @frame_io = nil
+            @logger.error("Recovery abandoned: #{e.message}")
+            @state = :closed
+            @recovery_in_progress = false
+            @channels.values.each { |ch| ch.mark_closed!(e) rescue nil }
+            instrument("recovery.exhausted") { { attempts: attempts, reason: :authentication_failed } }
+            @on_recovery_exhausted&.call(self)
+            @recovery_task = nil
+            return
+          rescue => e
+            @frame_io&.stop rescue nil
+            @logger.debug("Recovery: #{target_host}:#{target_port} failed — #{e.class}: #{e.message}")
+          end
+        end
+
+        if connected
+          # Only drops from here on belong to this connection: the old socket
+          # goes on failing throughout the backoff, and those errors must not
+          # make the reconnect we just made look like it had dropped too.
+          @recovery_interrupted = false
+          @heartbeat_task&.cancel rescue nil
+          start_heartbeat_task
+          start_channel0_monitor_task
+          @state = :open
+          @channel_ids = fresh_channel_ids
+          # NOTE: keep @recovery_task non-nil until reopen_channels completes so
+          # that session.close can still cancel this task (and therefore interrupt
+          # any wait_for calls inside reopen_after_recovery) if the user closes
+          # the session while channels are being reopened.
+          @logger.info("Recovery successful after #{attempts} attempt(s)")
+
+          # Re-open channels and re-register consumers. @recovery_in_progress
+          # stays true across this: clearing it first leaves a window where a
+          # second drop starts a competing recovery while this one is still
+          # failing channels through mark_closed!, which loses those channels
+          # and their consumers for good.
+          unless reopen_channels
+            # Dropped again mid-reopen. Channels are still :recovering, so go
+            # round again and reopen them on the next connection — but tear
+            # this half-built connection down first. Leaving it up reported the
+            # session as open (so a Cluster would place new channels on a dead
+            # node), held the socket, and leaked a channel-0 monitor per flap.
+            @logger.warn("Connection lost while reopening channels; retrying recovery")
+            @state = :recovering
+            @heartbeat_task&.cancel rescue nil
+            @heartbeat_task = nil
+            @channel0_task&.cancel rescue nil
+            @channel0_task = nil
+            @frame_io&.stop rescue nil
+            delay = [delay * 2, @recovery_max_interval].min
+            next
+          end
+
+          @recovery_in_progress = false
+          instrument("recovery.succeeded") do
+            { attempts: attempts, host: @host, port: @port, channels: @channels.size,
+              duration: recovery_started_at ? instrument_elapsed(recovery_started_at) : nil }
+          end
+          @on_recovery&.call(self)
+          @recovery_task = nil
+          return
+        else
+          @logger.warn("Recovery attempt #{attempts} failed: no reachable host")
+          delay = [delay * 2, @recovery_max_interval].min
+          break if @closed_by_user
+        end
+      end
+    ensure
+      # Make sure state is consistent if we exit for any reason
+      @recovery_in_progress = false if @closed_by_user
+    end
+
+    # After reconnect: reopen every channel, replay the recorded topology
+    # (session-wide, in dependency order), then release parked callers and
+    # re-register consumers.
+    # Returns false if the connection dropped again while channels were being
+    # reopened. Channels are then left :recovering rather than closed, so the
+    # next pass through recover_loop can reopen them; only a failure that is
+    # this channel's own (a broker rejection) closes it for good.
+    def reopen_channels
+      reopened = []
+
+      @channels.values.each do |channel|
+        begin
+          channel.reopen_on(@frame_io, state: :recovering)
+          reopened << channel
+        rescue => e
+          return false if @recovery_interrupted
+
+          # Fail the channel loudly rather than leave its parked callers hanging.
+          @logger.error("Channel #{channel.channel_id} could not be reopened after recovery: #{e.class}: #{e.message}")
+          channel.mark_closed!(ChannelError.new("Channel could not be reopened after recovery: #{e.message}",
+                                                channel_id: channel.channel_id))
+          channel_closed(channel.channel_id)
+        end
+      end
+
+      recover_topology_on(reopened) if @recover_topology && !@topology.empty?
+      return false if @recovery_interrupted
+
+      reopened.each { |channel| channel.finish_recovery! rescue nil }
+      true
+    end
+
+    # Re-declare exchanges, then queues, then bindings. Each entity is replayed
+    # on the channel that declared it if that channel is still open, otherwise
+    # on a temporary channel. One entity failing does not stop the others.
+    def recover_topology_on(channels)
+      by_id = channels.to_h { |ch| [ch.channel_id, ch] }
+      temp  = nil
+      on    = ->(channel_id) { by_id[channel_id] || (temp ||= open_channel) }
+
+      @topology.exchanges.each         { |x| on.call(x.channel_id).recover_exchange(x) }
+      @topology.queues.each            { |q| on.call(q.channel_id).recover_queue(q) }
+      @topology.queue_bindings.each    { |b| on.call(b.channel_id).recover_queue_binding(b) }
+      @topology.exchange_bindings.each { |b| on.call(b.channel_id).recover_exchange_binding(b) }
+    rescue => e
+      @logger.error("Topology recovery aborted: #{e.class}: #{e.message}")
+    ensure
+      temp&.close rescue nil
+    end
+
+
+    # Dedicated long-lived task that drains the channel-0 queue after the
+    # AMQP handshake completes.  Handles connection.blocked / connection.unblocked
+    # by delegating to FrameIO's blocked-state gate so write_frame yields
+    # automatically when the broker is resource-constrained.
+    def start_channel0_monitor_task
+      # Cancel any predecessor: a recovery that has to retry starts this again,
+      # and the old one would otherwise sit on a dead queue for good.
+      @channel0_task&.cancel rescue nil
+      @channel0_task = spawn_background { channel0_monitor_loop }
+    end
+
+    def channel0_monitor_loop
+      queue = @frame_io.channel_queue(0)
+      loop do
+        msg = queue.pop
+        break if msg.nil?
+        next unless msg[0] == :method
+        method = msg[1]
+        case method
+        when AMQ::Protocol::Connection::Blocked
+          @frame_io&.set_blocked(method.reason)
+          instrument("connection.blocked") { { reason: method.reason } }
+          @on_blocked&.call(method.reason)
+        when AMQ::Protocol::Connection::Unblocked
+          @frame_io&.set_unblocked
+          instrument("connection.unblocked") { {} }
+          @on_unblocked&.call
+        when AMQ::Protocol::Connection::UpdateSecretOk
+          @update_secret_condition&.signal(method)
+        when AMQ::Protocol::Connection::Close
+          # FrameIO has already answered with close-ok and triggered recovery;
+          # fail a pending update_secret with the broker's reason.
+          @update_secret_condition&.signal(ConnectionError.new(code: method.reply_code, text: method.reply_text))
+        end
+        # Other channel-0 methods during normal operation are intentionally
+        # ignored; the handshake uses wait_channel0_method, not this loop.
+      end
+    rescue => e
+      @logger.debug("Channel-0 monitor exited: #{e.class}: #{e.message}")
+    end
+  end
+end