hutch 1.4.0 → 2.0.0

data/lib/hutch/serializers/json.rb

@@ -1,4 +1,4 @@
-require 'multi_json'
+require 'json'
 require 'active_support/core_ext/hash/indifferent_access'
 
 module Hutch
@@ -6,11 +6,11 @@ module Hutch
     class JSON
 
       def self.encode(payload)
-        ::MultiJson.dump(payload)
+        ::JSON.generate(payload)
       end
 
       def self.decode(payload)
-        ::MultiJson.load(payload).with_indifferent_access
+        ::JSON.parse(payload).with_indifferent_access
       end
 
       def self.binary? ; false ; end

data/lib/hutch/tracers/datadog.rb

@@ -1,12 +1,5 @@
-begin
-  require 'datadog'
-  require 'datadog/auto_instrument'
-rescue LoadError
-  require 'ddtrace'
-  require 'ddtrace/auto_instrument'
-  warn "[DEPRECATION] The ddtrace gem is deprecated and Hutch will require the datadog gem in 2.0. " \
-       "Please switch to the datadog gem."
-end
+require 'datadog'
+require 'datadog/auto_instrument'
 
 module Hutch
   module Tracers

data/lib/hutch/version.rb

@@ -1,3 +1,3 @@
 module Hutch
-  VERSION = '1.4.0'.freeze
+  VERSION = '2.0.0'.freeze
 end

data/lib/hutch/worker.rb

@@ -11,8 +11,9 @@ module Hutch
     include Logging
 
     def initialize(broker, consumers, setup_procs)
-      @broker        = broker
-      self.consumers = consumers
+      @broker         = broker
+      @recovery_lock  = Mutex.new
+      self.consumers  = consumers
       self.setup_procs = setup_procs
     end
 
@@ -52,12 +53,46 @@ module Hutch
       queue = @broker.queue(queue_name, consumer.get_options)
       @broker.bind_queue(queue, consumer.routing_keys)
 
-      queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
+      on_cancellation = proc { handle_cancellation(queue_name, queue.channel) }
+      queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true,
+                      on_cancellation: on_cancellation) do |*args|
         delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
         handle_message(consumer, delivery_info, properties, payload)
       end
     end
 
+    # A server-sent `basic.cancel` carries no reason, so a consumer timeout
+    # is indistinguishable from a queue deletion. Only the former is recoverable.
+    def handle_cancellation(queue_name, cancelled_channel)
+      unless @broker.queue_exists?(queue_name)
+        logger.error "consumer on queue #{queue_name} was cancelled by the server: the queue no longer exists"
+        return
+      end
+
+      logger.warn "consumer on queue #{queue_name} was cancelled by the server, re-subscribing"
+      resubscribe_on_a_new_channel(cancelled_channel)
+    rescue => ex
+      logger.error "consumer re-subscription failed: #{ex.class}: #{ex.message}"
+    end
+
+    # Runs on its own thread: closing the channel kills the consumer work
+    # pool this callback runs on. Returns the thread so that tests can join it.
+    def resubscribe_on_a_new_channel(cancelled_channel)
+      Thread.new do
+        begin
+          @recovery_lock.synchronize do
+            # All consumers share the channel, so only the first cancellation replaces it.
+            next unless @broker.channel.equal?(cancelled_channel)
+
+            @broker.replace_channel!
+            setup_queues
+          end
+        rescue => ex
+          logger.error "consumer re-subscription failed: #{ex.class}: #{ex.message}"
+        end
+      end
+    end
+
     # Called internally when a new messages comes in from RabbitMQ. Responsible
     # for wrapping up the message and passing it to the consumer.
     def handle_message(consumer, delivery_info, properties, payload)
@@ -73,7 +108,7 @@ module Hutch
       message = Message.new(delivery_info, properties, payload, serializer)
       consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
       with_tracing(consumer_instance).handle(message)
-      @broker.ack(delivery_info.delivery_tag) unless consumer_instance.message_rejected?
+      @broker.ack(delivery_info.delivery_tag, channel: delivery_info.channel) unless consumer_instance.message_rejected?
     rescue => ex
       acknowledge_error(delivery_info, properties, @broker, ex)
       handle_error(properties, payload, consumer, ex, delivery_info)

data/lib/yard-settings/yard-settings.rb

@@ -1,2 +1,2 @@
-YARD::Templates::Engine.register_template_path(File.dirname(__FILE__) + '/../../templates')
-require File.join(File.dirname(__FILE__), 'handler') if RUBY19
+YARD::Templates::Engine.register_template_path(File.join(__dir__, '../../templates'))
+require_relative 'handler'

data/spec/hutch/broker_spec.rb

@@ -128,6 +128,76 @@ describe Hutch::Broker do
     end
   end
 
+  describe '#parse_uri' do
+    {
+      "amqp://host/vhost"    => "vhost",
+      "amqp://host/v%2fhost" => "v/host",
+      "amqp://host/%2f"      => "/",
+      "amqp://host/"         => Hutch::Adapter::DEFAULT_VHOST,
+      "amqp://host"          => Hutch::Adapter::DEFAULT_VHOST,
+    }.each do |uri, vhost|
+      it "parses the vhost of #{uri} as #{vhost.inspect}" do
+        config[:uri] = uri
+        broker.send(:parse_uri)
+
+        expect(config[:mq_vhost]).to eq(vhost)
+      end
+    end
+
+    it 'passes the vhost to the connection unchanged' do
+      config[:uri] = "amqp://host/%2f"
+
+      expect(broker.send(:connection_params)[:vhost]).to eq("/")
+    end
+
+    it 'percent-decodes the username and password' do
+      config[:uri] = "amqp://al%23pha:be%20ta@host:10000/vhost"
+      broker.send(:parse_uri)
+
+      expect(config[:mq_username]).to eq("al#pha")
+      expect(config[:mq_password]).to eq("be ta")
+    end
+
+    it 'leaves absent credentials nil' do
+      config[:uri] = "amqp://host:10000/vhost"
+      broker.send(:parse_uri)
+
+      expect(config[:mq_username]).to be_nil
+      expect(config[:mq_password]).to be_nil
+    end
+  end
+
+  describe '#bindings', 'vhost filtering' do
+    let(:api_bindings) do
+      [{ 'source' => 'hutch', 'vhost' => '/', 'destination' => 'q', 'routing_key' => 'key' }]
+    end
+
+    before do
+      config[:mq_exchange] = 'hutch'
+      broker.api_client = double('api_client', bindings: api_bindings)
+    end
+
+    it 'filters on the default vhost when the configured vhost is blank' do
+      config[:mq_vhost] = ''
+
+      expect(broker.bindings).to eq('q' => ['key'])
+    end
+  end
+
+  describe '#sanitized_uri' do
+    it 'percent-encodes the decoded username and vhost again' do
+      config[:uri] = 'amqp://al%23pha:be%20ta@host:10000/v%2Fhost'
+
+      expect(broker.send(:sanitized_uri)).to eq('amqp://al%23pha@host:10000/v%2Fhost')
+    end
+
+    it 'renders the default vhost as a bare trailing slash' do
+      config[:uri] = 'amqp://alpha:beta@host:10000/'
+
+      expect(broker.send(:sanitized_uri)).to eq('amqp://alpha@host:10000/')
+    end
+  end
+
   describe '#open_connection!' do
     it 'sets the #connection to #open_connection' do
       connection = double('connection').as_null_object
@@ -199,6 +269,100 @@ describe Hutch::Broker do
     end
   end
 
+  describe '#replace_channel!', adapter: :bunny do
+    let(:old_channel) { double('Bunny::Channel', open?: true, close: nil) }
+    let(:new_channel) { double('Bunny::Channel').as_null_object }
+    let(:work_pool) { double('Bunny::ConsumerWorkPool', shutdown: nil, join: nil, kill: nil) }
+
+    before do
+      broker.channel = old_channel
+      allow(broker).to receive(:channel_work_pool).and_return(work_pool)
+      allow(broker).to receive(:open_channel).and_return(new_channel)
+      allow(broker).to receive(:declare_exchange!)
+      allow(broker).to receive(:declare_publisher!)
+    end
+
+    it 'drains the work pool before closing the old channel' do
+      expect(work_pool).to receive(:shutdown).ordered
+      expect(work_pool).to receive(:join).with(config[:graceful_exit_timeout]).ordered
+      expect(work_pool).to receive(:kill).ordered
+      expect(old_channel).to receive(:close).ordered
+
+      broker.replace_channel!
+    end
+
+    it 'redeclares the exchange and the publisher on the new channel' do
+      expect(broker).to receive(:declare_exchange!)
+      expect(broker).to receive(:declare_publisher!)
+
+      broker.replace_channel!
+
+      expect(broker.channel).to eq(new_channel)
+    end
+
+    context 'when the old channel is already closed' do
+      let(:old_channel) { double('Bunny::Channel', open?: false) }
+
+      it 'opens a new channel anyway' do
+        expect(old_channel).not_to receive(:close)
+        expect(work_pool).not_to receive(:shutdown)
+
+        broker.replace_channel!
+
+        expect(broker.channel).to eq(new_channel)
+      end
+    end
+
+    context 'when the server closes the old channel concurrently' do
+      it 'opens a new channel anyway' do
+        allow(old_channel).to receive(:close).
+          and_raise(Hutch::Adapter::ChannelAlreadyClosed.new('already closed', old_channel))
+
+        broker.replace_channel!
+
+        expect(broker.channel).to eq(new_channel)
+      end
+    end
+  end
+
+  describe 'acknowledgements' do
+    let(:current_channel) { double('Channel').as_null_object }
+    let(:superseded_channel) { double('Channel') }
+
+    before { broker.channel = current_channel }
+
+    it 'sends them on the channel the delivery arrived on' do
+      expect(current_channel).to receive(:ack).with('dt', false)
+      broker.ack('dt', channel: current_channel)
+
+      expect(current_channel).to receive(:nack).with('dt', false, false)
+      broker.nack('dt', channel: current_channel)
+
+      expect(current_channel).to receive(:reject).with('dt', false)
+      broker.reject('dt', channel: current_channel)
+
+      expect(current_channel).to receive(:reject).with('dt', true)
+      broker.requeue('dt', channel: current_channel)
+    end
+
+    it 'drops them when the channel has been replaced' do
+      expect(superseded_channel).not_to receive(:ack)
+      expect(superseded_channel).not_to receive(:nack)
+      expect(superseded_channel).not_to receive(:reject)
+
+      broker.ack('dt', channel: superseded_channel)
+      broker.nack('dt', channel: superseded_channel)
+      broker.reject('dt', channel: superseded_channel)
+      broker.requeue('dt', channel: superseded_channel)
+    end
+
+    it 'defaults to the current channel' do
+      expect(current_channel).to receive(:ack).with('dt', false)
+
+      broker.ack('dt')
+    end
+  end
+
   describe '#declare_exchange' do
     before do
       broker.open_connection!
@@ -293,7 +457,7 @@ describe Hutch::Broker do
 
     context 'with a binding' do
       around do |example|
-        queue = broker.queue('test').bind(broker.exchange, routing_key: 'key')
+        queue = broker.queue('test', durable: true).bind(broker.exchange, routing_key: 'key')
         example.run
         queue.unbind(broker.exchange, routing_key: 'key').delete
       end
@@ -323,7 +487,7 @@ describe Hutch::Broker do
     end
 
     context '(rabbitmq integration test)', rabbitmq: true do
-      let(:queue) { broker.queue('consumer') }
+      let(:queue) { broker.queue('consumer', durable: true) }
       let(:routing_key) { 'key' }
 
       before { allow(broker).to receive(:bindings).and_call_original }

data/spec/hutch/cli_spec.rb

@@ -31,9 +31,10 @@ describe Hutch::CLI do
       end
 
       context "when the config file exists" do
-        let(:file) do
-          Tempfile.new("hutch-test-config.yaml").to_path
-        end
+        # Kept in a let, so the Tempfile is not garbage collected - and with it
+        # unlinked - before the CLI checks that the path exists.
+        let(:config_file) { Tempfile.new("hutch-test-config.yaml") }
+        let(:file) { config_file.to_path }
 
         it "parses the config" do
           expect(Hutch::Config).to receive(:load_from_file)
@@ -55,9 +56,10 @@ describe Hutch::CLI do
       end
 
       context "when the keyfile file exists" do
-        let(:file) do
-          Tempfile.new("hutch-test-key.pem").to_path
-        end
+        # Kept in a let, so the Tempfile is not garbage collected - and with it
+        # unlinked - before the CLI checks that the path exists.
+        let(:key_file) { Tempfile.new("hutch-test-key.pem") }
+        let(:file) { key_file.to_path }
 
         it "sets mq_tls_key to the file" do
           expect(Hutch::Config).to receive(:mq_tls_key=)
@@ -79,9 +81,10 @@ describe Hutch::CLI do
       end
 
       context "when the certfile file exists" do
-        let(:file) do
-          Tempfile.new("hutch-test-cert.pem").to_path
-        end
+        # Kept in a let, so the Tempfile is not garbage collected - and with it
+        # unlinked - before the CLI checks that the path exists.
+        let(:cert_file) { Tempfile.new("hutch-test-cert.pem") }
+        let(:file) { cert_file.to_path }
 
         it "sets mq_tls_cert to the file" do
           expect(Hutch::Config).to receive(:mq_tls_cert=)

data/spec/hutch/error_handlers/airbrake_spec.rb

@@ -1,5 +1,8 @@
 require 'spec_helper'
 
+# The airbrake gem is MRI-only, see the Gemfile
+return if defined?(JRUBY_VERSION)
+
 describe Hutch::ErrorHandlers::Airbrake do
   let(:error_handler) { Hutch::ErrorHandlers::Airbrake.new }
 

data/spec/hutch/message_spec.rb

@@ -4,7 +4,7 @@ describe Hutch::Message do
   let(:delivery_info) { double('Delivery Info') }
   let(:props) { double('Properties', content_type: "application/json") }
   let(:body) {{ foo: 'bar' }.with_indifferent_access}
-  let(:json_body) { MultiJson.dump(body) }
+  let(:json_body) { JSON.generate(body) }
   subject(:message) { Hutch::Message.new(delivery_info, props, json_body, Hutch::Config[:serializer]) }
 
   describe '#body' do

data/spec/hutch/worker_spec.rb

@@ -60,7 +60,7 @@ describe Hutch::Worker do
     end
 
     it 'sets up a subscription' do
-      expect(queue).to receive(:subscribe).with(consumer_tag: %r(^hutch\-.{36}$), manual_ack: true)
+      expect(queue).to receive(:subscribe).with(consumer_tag: %r(^hutch\-.{36}$), manual_ack: true, on_cancellation: kind_of(Proc))
       worker.setup_queue(consumer)
     end
 
@@ -68,7 +68,7 @@ describe Hutch::Worker do
       before { Hutch::Config.set(:consumer_tag_prefix, 'appname') }
 
       it 'sets up a subscription with the configured tag prefix' do
-        expect(queue).to receive(:subscribe).with(consumer_tag: %r(^appname\-.{36}$), manual_ack: true)
+        expect(queue).to receive(:subscribe).with(consumer_tag: %r(^appname\-.{36}$), manual_ack: true, on_cancellation: kind_of(Proc))
         worker.setup_queue(consumer)
       end
     end
@@ -84,11 +84,65 @@ describe Hutch::Worker do
     end
   end
 
+  describe '#handle_cancellation' do
+    let(:cancelled_channel) { double('Channel') }
+    let(:new_channel) { double('Channel') }
+    let(:log) { StringIO.new }
+
+    before do
+      allow(Hutch::Logging).to receive(:logger).and_return(Logger.new(log))
+      allow(broker).to receive(:channel).and_return(cancelled_channel)
+      allow(broker).to receive(:replace_channel!) do
+        allow(broker).to receive(:channel).and_return(new_channel)
+      end
+      allow(worker).to receive(:setup_queues)
+    end
+
+    context 'when the queue still exists' do
+      before { allow(broker).to receive(:queue_exists?).with('consumer').and_return(true) }
+
+      it 'replaces the channel and re-subscribes' do
+        expect(broker).to receive(:replace_channel!)
+        expect(worker).to receive(:setup_queues)
+
+        worker.handle_cancellation('consumer', cancelled_channel).join
+      end
+
+      it 'replaces the channel once for all the consumers that shared it' do
+        expect(broker).to receive(:replace_channel!).once
+
+        [
+          worker.handle_cancellation('consumer', cancelled_channel),
+          worker.handle_cancellation('consumer', cancelled_channel)
+        ].each(&:join)
+      end
+
+      it 'does not replace a channel that has already been replaced' do
+        expect(broker).not_to receive(:replace_channel!)
+
+        worker.handle_cancellation('consumer', double('An older channel')).join
+      end
+    end
+
+    context 'when the queue is gone' do
+      before { allow(broker).to receive(:queue_exists?).with('consumer').and_return(false) }
+
+      it 'does not re-subscribe' do
+        expect(broker).not_to receive(:replace_channel!)
+
+        worker.handle_cancellation('consumer', cancelled_channel)
+
+        log.rewind
+        expect(log.read).to match(/the queue no longer exists/)
+      end
+    end
+  end
+
   describe '#handle_message' do
     subject { worker.handle_message(consumer, delivery_info, properties, payload) }
     let(:payload) { '{}' }
     let(:consumer_instance) { double('Consumer instance') }
-    let(:delivery_info) { double('Delivery Info', routing_key: '',
+    let(:delivery_info) { double('Delivery Info', routing_key: '', channel: nil,
                                  delivery_tag: 'dt') }
     let(:properties) { double('Properties', message_id: nil, content_type: "application/json") }
     let(:log) { StringIO.new }
@@ -108,7 +162,7 @@ describe Hutch::Worker do
 
     it 'acknowledges the message' do
       allow(consumer_instance).to receive(:process)
-      expect(broker).to receive(:ack).with(delivery_info.delivery_tag)
+      expect(broker).to receive(:ack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
       expect(consumer_instance).to receive(:message_rejected?).and_return(false)
       subject
     end
@@ -141,7 +195,7 @@ describe Hutch::Worker do
       end
 
       it 'rejects the message' do
-        expect(broker).to receive(:nack).with(delivery_info.delivery_tag)
+        expect(broker).to receive(:nack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
         subject
       end
 
@@ -191,14 +245,14 @@ describe Hutch::Worker do
 
     context "when the payload is not valid json" do
       let(:payload) { "Not Valid JSON" }
-      let(:expected_log) { /ERROR .+ error in consumer .+ MultiJson::ParseError .+ backtrace:/m }
+      let(:expected_log) { /ERROR .+ error in consumer .+ JSON::ParserError .+ backtrace:/m }
 
       it 'logs the error' do
         expect { subject }.to change { log.tap(&:rewind).read }.from("").to(expected_log)
       end
 
       it 'rejects the message' do
-        expect(broker).to receive(:nack).with(delivery_info.delivery_tag)
+        expect(broker).to receive(:nack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
         subject
       end
     end
@@ -206,7 +260,7 @@ describe Hutch::Worker do
 
 
   describe '#acknowledge_error' do
-    let(:delivery_info) { double('Delivery Info', routing_key: '',
+    let(:delivery_info) { double('Delivery Info', routing_key: '', channel: nil,
                                  delivery_tag: 'dt') }
     let(:properties) { double('Properties', message_id: 'abc123') }
 

data/spec/integration/channel_recovery_spec.rb

@@ -1,13 +1,22 @@
 require 'spec_helper'
+
+# Channel recovery is bunny-only, see MarchHareAdapter#install_channel_recovery
+return if defined?(JRUBY_VERSION)
 require 'hutch/broker'
 require 'hutch/worker'
 require 'hutch/consumer'
 require 'bunny'
 require 'json'
 require 'securerandom'
-require 'timeout'
 
 describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: true, adapter: :bunny do
+  CONSUMER_TIMEOUT_MS = 5_000
+  # A 4.3 quorum queue times the consumer out on its own timer, which fires at
+  # the deadline. Earlier series detect it from the channel tick instead, and
+  # `channel_tick_interval` defaults to 60s, so detection lags by up to that much.
+  BLOCKED_HANDLER_SECONDS = 20
+  BLOCKED_HANDLER_SECONDS_BEFORE_4_3 = 100
+
   let(:log) { StringIO.new }
   let(:logger) { Logger.new(log) }
   let(:exchange_name) { "hutch.integration.exchange.#{SecureRandom.hex(4)}" }
@@ -18,12 +27,25 @@ describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: tr
   let(:processed_lock) { Mutex.new }
   let(:timed_out_once) { [false] }
 
+  let(:server_version) do
+    Gem::Version.new(publisher.server_properties['version'].to_s[/\A\d+(\.\d+)*/])
+  end
+
+  let(:blocked_handler_seconds) do
+    if server_version >= Gem::Version.new('4.3')
+      BLOCKED_HANDLER_SECONDS
+    else
+      BLOCKED_HANDLER_SECONDS_BEFORE_4_3
+    end
+  end
+
   let(:consumer_class) do
     msgs = processed
     lock = processed_lock
     rk = routing_key
     qn = queue_name
     timed_out = timed_out_once
+    blocked = blocked_handler_seconds
 
     Class.new do
       include Hutch::Consumer
@@ -32,13 +54,15 @@ describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: tr
       queue_name qn
       arguments(
         'x-queue-type' => 'quorum',
-        'x-consumer-timeout' => 60_000
+        'x-consumer-timeout' => CONSUMER_TIMEOUT_MS
       )
 
+      # Both `CONSUMER_TIMEOUT_MS` and this sleep run from the same delivery,
+      # so the margin between them does not shrink on a loaded machine.
       define_method(:process) do |message|
         if message['id'] == 'trigger-timeout' && !timed_out[0]
           timed_out[0] = true
-          sleep 210
+          sleep blocked
         end
 
         lock.synchronize { msgs << message['id'] }
@@ -67,25 +91,25 @@ describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: tr
     Hutch::Config.set(:mq_exchange, exchange_name)
     Hutch::Config.set(:force_publisher_confirms, false)
     Hutch::Config.set(:client_logger, logger)
+    # Channel replacement waits this long for the deliberately blocked
+    # handler: the default of 11s would use up most of the consumption window.
+    @graceful_exit_timeout = Hutch::Config[:graceful_exit_timeout]
+    Hutch::Config.set(:graceful_exit_timeout, 1)
   end
 
   after do
     publisher_channel.close rescue nil
     publisher.close rescue nil
     broker.disconnect rescue nil
+    Hutch::Config.set(:graceful_exit_timeout, @graceful_exit_timeout)
     Hutch::Logging.logger = Logger.new(File::NULL)
   end
 
-  def wait_for(timeout, label)
-    Timeout.timeout(timeout) do
-      loop do
-        return true if yield
-        sleep 0.25
-      end
-    end
-  rescue Timeout::Error
+  def wait_for(timeout, label, &condition)
+    await_condition(timeout, label, &condition)
+  rescue AwaitHelpers::ConditionTimeout => e
     raise <<~MSG
-      Timed out waiting for: #{label}
+      #{e.message}
 
       processed_messages=#{processed_messages.inspect}
       channel_open=#{broker.channel.open? rescue 'unknown'}
@@ -100,9 +124,10 @@ describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: tr
     processed_lock.synchronize { processed.dup }
   end
 
+  # `StringIO#string` does not touch the stream position: a rewind here would
+  # race with the worker threads that log concurrently and corrupt the buffer.
   def log_output
-    log.rewind
-    log.read
+    log.string
   end
 
   def publish_message(id)
@@ -114,26 +139,49 @@ describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: tr
     )
   end
 
+  # RabbitMQ up to 4.2 closes the channel on a delivery acknowledgement
+  # timeout; 4.3+ quorum queues instead cancel only the timed out consumer.
+  let(:cancels_the_consumer) { server_version >= Gem::Version.new('4.3') }
+
+  let(:recovery_log_pattern) do
+    if cancels_the_consumer
+      /cancelled by the server, re-subscribing/i
+    else
+      /delivery acknowledgement on channel \d+ timed out/i
+    end
+  end
+
+  # Replacing the channel requeues what the cancelled consumer still held, so
+  # consumption resumes at once. Reopening one does not, so it resumes at the
+  # next consumer timeout tick.
+  let(:recovery_seconds) { cancels_the_consumer ? 15 : 90 }
+
   # This spec is intentionally slow because RabbitMQ enforces delivery
-  # acknowledgement timeouts on a periodic sweep, not immediately at the deadline.
-  it 're-subscribes and consumes later messages after RabbitMQ closes the channel for ack timeout' do
+  # acknowledgement timeouts periodically on a timer, not immediately at the deadline.
+  it 're-subscribes and consumes later messages after RabbitMQ times out an unacknowledged delivery' do
     broker.connect
     worker.setup_queues
 
     publish_message('trigger-timeout')
 
-    wait_for(240, 'delivery acknowledgement timeout') do
-      log_output.match?(/delivery acknowledgement on channel \d+ timed out/i)
+    wait_for(240, 'a delivery acknowledgement timeout to close the channel or cancel the consumer') do
+      log_output.match?(recovery_log_pattern)
     end
 
     publish_message('after-recovery')
 
-    wait_for(90, 'after-recovery message consumption') do
+    wait_for(recovery_seconds, 'after-recovery message consumption') do
       processed_messages.include?('after-recovery')
     end
 
-    expect(log_output).to match(/delivery acknowledgement on channel \d+ timed out/i)
-    expect(log_output).to match(/recovered consumer channel after a delivery acknowledgement timeout/i)
-    expect(processed_messages).to include('after-recovery')
+    expect(log_output).to match(recovery_log_pattern)
+
+    # A consumed message proves the recovery finished, the publisher rebuild
+    # included.
+    broker.publish(routing_key, 'id' => 'published-after-recovery')
+
+    wait_for(15, 'published-after-recovery message consumption') do
+      processed_messages.include?('published-after-recovery')
+    end
   end
-end
+end