bunny 1.3.0 → 2.17.0

data/spec/higher_level_api/integration/basic_return_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Bunny::Exchange, "#publish" do
   let(:connection) do
-    c = Bunny.new(:user => "bunny_gem", :password => "bunny_password", :vhost => "bunny_testbed")
+    c = Bunny.new(username: "bunny_gem", password: "bunny_password", vhost: "bunny_testbed")
     c.start
     c
   end
@@ -12,7 +12,7 @@ describe Bunny::Exchange, "#publish" do
   end
 
 
-  context "with :mandatory => true and a bad [no routes] routing key" do
+  context "with mandatory: true and a bad [no routes] routing key" do
     it "causes a message to be returned" do
       ch = connection.create_channel
       x  = ch.default_exchange
@@ -22,10 +22,10 @@ describe Bunny::Exchange, "#publish" do
         returned << content
       end
 
-      x.publish("xyzzy", :routing_key => rand.to_s, :mandatory => true)
+      x.publish("xyzzy", routing_key: rand.to_s, mandatory: true)
       sleep 0.5
 
-      returned.should include("xyzzy")
+      expect(returned).to include("xyzzy")
 
       ch.close
     end

data/spec/higher_level_api/integration/channel_close_spec.rb

@@ -1,8 +1,8 @@
 require "spec_helper"
 
-describe Bunny::Channel, "when closed" do
+describe Bunny::Channel do
   let(:connection) do
-    c = Bunny.new(:user => "bunny_gem", :password => "bunny_password", :vhost => "bunny_testbed")
+    c = Bunny.new(username: "bunny_gem", password: "bunny_password", vhost: "bunny_testbed")
     c.start
     c
   end
@@ -11,15 +11,56 @@ describe Bunny::Channel, "when closed" do
     connection.close
   end
 
-  it "releases the id" do
-    ch = connection.create_channel
-    n = ch.number
+  context "when closed" do
+    it "releases the id" do
+      ch = connection.create_channel
+      n = ch.number
 
-    ch.should be_open
-    ch.close
-    ch.should be_closed
+      expect(ch).to be_open
+      ch.close
+      expect(ch).to be_closed
 
-    # a new channel with the same id can be created
-    connection.create_channel(n)
+      # a new channel with the same id can be created
+      connection.create_channel(n)
+    end
+  end
+
+  context "when double closed" do
+    # bunny#528
+    it "raises a meaningful exception" do
+      ch = connection.create_channel
+
+      expect(ch).to be_open
+      ch.close
+      expect(ch).to be_closed
+
+      expect { ch.close }.to raise_error(Bunny::ChannelAlreadyClosed)
+    end
+  end
+
+  context "when double closed after a channel-level protocol exception" do
+    # bunny#528
+    it "raises a meaningful exception" do
+      ch = connection.create_channel
+
+      s  = "bunny-temp-q-#{rand}"
+
+      expect(ch).to be_open
+      ch.queue_declare(s, durable: false)
+
+      expect do
+        ch.queue_declare(s, durable: true)
+      end.to raise_error(Bunny::PreconditionFailed)
+
+      # channel.close is sent and handled concurrently with the test
+      sleep 1
+      expect(ch).to be_closed
+
+      expect { ch.close }.to raise_error(Bunny::ChannelAlreadyClosed)
+
+      cleanup_ch = connection.create_channel
+      cleanup_ch.queue_delete(s)
+      cleanup_ch.close
+    end
   end
 end

data/spec/higher_level_api/integration/channel_open_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Bunny::Channel, "when opened" do
   let(:connection) do
-    c = Bunny.new(:user => "bunny_gem", :password => "bunny_password", :vhost => "bunny_testbed")
+    c = Bunny.new(username: "bunny_gem", password: "bunny_password", vhost: "bunny_testbed")
     c.start
     c
   end
@@ -13,17 +13,17 @@ describe Bunny::Channel, "when opened" do
 
   context "without explicitly provided id" do
     it "gets an allocated id and is successfully opened" do
-      connection.should be_connected
+      expect(connection).to be_connected
       ch = connection.create_channel
-      ch.should be_open
+      expect(ch).to be_open
 
-      ch.id.should be > 0
+      expect(ch.id).to be > 0
     end
   end
 
   context "with an explicitly provided id = 0" do
     it "raises ArgumentError" do
-      connection.should be_connected
+      expect(connection).to be_connected
       expect {
         connection.create_channel(0)
       }.to raise_error(ArgumentError)
@@ -34,10 +34,10 @@ describe Bunny::Channel, "when opened" do
   context "with explicitly provided id" do
     it "uses that id and is successfully opened" do
       ch = connection.create_channel(767)
-      connection.should be_connected
-      ch.should be_open
+      expect(connection).to be_connected
+      expect(ch).to be_open
 
-      ch.id.should == 767
+      expect(ch.id).to eq 767
     end
   end
 
@@ -46,12 +46,12 @@ describe Bunny::Channel, "when opened" do
   context "with explicitly provided id that is already taken" do
     it "reuses the channel that is already opened" do
       ch = connection.create_channel(767)
-      connection.should be_connected
-      ch.should be_open
+      expect(connection).to be_connected
+      expect(ch).to be_open
 
-      ch.id.should == 767
+      expect(ch.id).to eq 767
 
-      connection.create_channel(767).should == ch
+      expect(connection.create_channel(767)).to eq ch
     end
   end
 end

data/spec/higher_level_api/integration/connection_recovery_spec.rb

@@ -1,280 +1,483 @@
 require "spec_helper"
 require "rabbitmq/http/client"
 
-unless ENV["CI"]
-  describe "Connection recovery" do
-    let(:connection)  {  }
-    let(:http_client) { RabbitMQ::HTTP::Client.new("http://127.0.0.1:15672") }
-
-    def close_all_connections!
-      http_client.list_connections.each do |conn_info|
-        http_client.close_connection(conn_info.name)
-      end
+require "bunny/concurrent/condition"
+
+describe "Connection recovery" do
+  let(:http_client) { RabbitMQ::HTTP::Client.new("http://127.0.0.1:15672") }
+  let(:logger) { Logger.new($stderr).tap {|logger| logger.level = ENV["BUNNY_LOG_LEVEL"] || Logger::WARN } }
+  let(:recovery_interval) { 0.2 }
+
+  it "reconnects after grace period" do
+    with_open do |c|
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+    end
+  end
+
+  it "reconnects after grace period (with multiple hosts)" do
+    with_open_multi_host do |c|
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+    end
+  end
+
+  it "reconnects after grace period (with multiple hosts, including a broken one)" do
+    with_open_multi_broken_host do |c|
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
     end
+  end
 
-    def wait_for_recovery
+  it "recovers channels" do
+    with_open do |c|
+      ch1 = c.create_channel
+      ch2 = c.create_channel
       sleep 1.5
+      close_all_connections!
+      sleep 0.5
+      poll_until { channels.count == 2 }
+      expect(ch1).to be_open
+      expect(ch2).to be_open
     end
+  end
 
-    def with_open(c = Bunny.new(:network_recovery_interval => 0.2, :recover_from_connection_close => true), &block)
-      begin
-        c.start
-        block.call(c)
-      ensure
-        c.close
+  it "provides a recovery completion callback" do
+    with_open do |c|
+      latch = Bunny::Concurrent::Condition.new
+      c.after_recovery_completed do
+        latch.notify
       end
+
+      ch = c.create_channel
+      sleep 1.0
+      close_all_connections!
+      poll_until { c.open? && ch.open? }
+      poll_until { latch.none_threads_waiting? }
+    end
+  end
+
+  it "recovers channels (with multiple hosts)" do
+    with_open_multi_host do |c|
+      ch1 = c.create_channel
+      ch2 = c.create_channel
+      sleep 1.5
+      close_all_connections!
+      sleep 0.5
+      poll_until { channels.count == 2 }
+      expect(ch1).to be_open
+      expect(ch2).to be_open
+    end
+  end
+
+  it "recovers channels (with multiple hosts, including a broken one)" do
+    with_open_multi_broken_host do |c|
+      ch1 = c.create_channel
+      ch2 = c.create_channel
+      sleep 1.5
+      close_all_connections!
+      sleep 0.5
+      poll_until { channels.count == 2 }
+      expect(ch1).to be_open
+      expect(ch2).to be_open
+    end
+  end
+
+  it "recovers basic.qos prefetch setting" do
+    with_open do |c|
+      ch = c.create_channel
+      ch.prefetch(11)
+      expect(ch.prefetch_count).to eq 11
+      expect(ch.prefetch_global).to be false
+      sleep 1.5
+      close_all_connections!
+      sleep 0.5
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      expect(ch.prefetch_count).to eq 11
+      expect(ch.prefetch_global).to be false
+    end
+  end
+
+  it "recovers basic.qos prefetch global setting" do
+    with_open do |c|
+      ch = c.create_channel
+      ch.prefetch(42, true)
+      expect(ch.prefetch_count).to eq 42
+      expect(ch.prefetch_global).to be true
+      sleep 1.5
+      close_all_connections!
+      sleep 0.5
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      expect(ch.prefetch_count).to eq 42
+      expect(ch.prefetch_global).to be true
     end
+  end
 
-    def ensure_queue_recovery(ch, q)
-      q.purge
-      x = ch.default_exchange
-      x.publish("msg", :routing_key => q.name)
+  it "recovers publisher confirms setting" do
+    with_open do |c|
+      ch = c.create_channel
+      ch.confirm_select
+      expect(ch).to be_using_publisher_confirms
+      sleep 1.5
+      close_all_connections!
       sleep 0.5
-      q.message_count.should == 1
-      q.purge
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      expect(ch).to be_using_publisher_confirms
     end
+  end
 
-    def ensure_queue_binding_recovery(x, q, routing_key = "")
-      q.purge
-      x.publish("msg", :routing_key => routing_key)
+  it "recovers transactionality setting" do
+    with_open do |c|
+      ch = c.create_channel
+      ch.tx_select
+      expect(ch).to be_using_tx
+      sleep 1.5
+      close_all_connections!
       sleep 0.5
-      q.message_count.should == 1
-      q.purge
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      expect(ch).to be_using_tx
+    end
+  end
+
+  it "recovers client-named queues" do
+    with_open do |c|
+      ch = c.create_channel
+      q  = ch.queue("bunny.tests.recovery.client-named#{rand}")
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      ensure_queue_recovery(ch, q)
+      q.delete
     end
+  end
+
+  # a very simplistic test for queues inspired by #422
+  it "recovers client-named queues declared with no_declare: true" do
+    with_open do |c|
+      ch  = c.create_channel
+      ch2 = c.create_channel
 
-    def ensure_exchange_binding_recovery(ch, source, destination, routing_key = "")
-      q  = ch.queue("", :exclusive => true)
-      q.bind(destination, :routing_key => routing_key)
+      n   = rand
+      s   = "bunny.tests.recovery.client-named#{n}"
 
-      source.publish("msg", :routing_key => routing_key)
-      q.message_count.should == 1
+      q   = ch.queue(s)
+      q2  = ch2.queue(s, no_declare: true)
+
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      ensure_queue_recovery(ch, q)
       q.delete
     end
+  end
 
-    #
-    # Examples
-    #
+  # a test for #422
+  it "recovers client-named queues declared with passive: true" do
+    with_open do |c|
+      ch  = c.create_channel
+      ch2 = c.create_channel
 
-    it "reconnects after grace period" do
-      with_open do |c|
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
+      n   = rand
+      s   = "bunny.tests.recovery.client-named#{n}"
 
-        wait_for_recovery
-        c.should be_open
-      end
+      q   = ch.queue(s)
+      q2  = ch2.queue(s, passive: true)
+
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      ensure_queue_recovery(ch, q)
+      ensure_queue_recovery(ch, q2)
+      q.delete
     end
+  end
 
-    it "recovers channel" do
-      with_open do |c|
-        ch1 = c.create_channel
-        ch2 = c.create_channel
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch1.should be_open
-        ch2.should be_open
-      end
+  it "recovers server-named queues" do
+    with_open do |c|
+      ch = c.create_channel
+      q  = ch.queue("", exclusive: true)
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      ensure_queue_recovery(ch, q)
     end
+  end
 
-    it "recovers basic.qos prefetch setting" do
-      with_open do |c|
-        ch = c.create_channel
-        ch.prefetch(11)
-        ch.prefetch_count.should == 11
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-        ch.prefetch_count.should == 11
-      end
+  it "recovers queue bindings" do
+    with_open do |c|
+      ch = c.create_channel
+      x  = ch.fanout("amq.fanout")
+      q  = ch.queue("", exclusive: true)
+      q.bind(x)
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      ensure_queue_binding_recovery(ch, x, q)
     end
+  end
 
+  it "recovers exchanges and their bindings" do
+    with_open do |c|
+      ch          = c.create_channel
+      source      = ch.fanout("source.exchange.recovery.example", auto_delete: true)
+      destination = ch.fanout("destination.exchange.recovery.example", auto_delete: true)
 
-    it "recovers publisher confirms setting" do
-      with_open do |c|
-        ch = c.create_channel
-        ch.confirm_select
-        ch.should be_using_publisher_confirms
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
+      destination.bind(source)
 
-        wait_for_recovery
-        ch.should be_open
-        ch.should be_using_publisher_confirms
-      end
-    end
+      # Exchanges won't get auto-deleted on connection loss unless they have
+      # had an exclusive queue bound to them.
+      dst_queue   = ch.queue("", exclusive: true)
+      dst_queue.bind(destination, routing_key: "")
 
-    it "recovers transactionality setting" do
-      with_open do |c|
-        ch = c.create_channel
-        ch.tx_select
-        ch.should be_using_tx
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-        ch.should be_using_tx
-      end
+      src_queue   = ch.queue("", exclusive: true)
+      src_queue.bind(source, routing_key: "")
+
+      close_all_connections!
+
+      wait_for_recovery_with { connections.any? && exchange_names_in_vhost("/").include?(source.name) }
+      ch.confirm_select
+
+      source.publish("msg", routing_key: "")
+      ch.wait_for_confirms
+      expect(dst_queue.message_count).to eq 1
+      destination.delete
     end
+  end
 
-    it "recovers client-named queues" do
-      with_open do |c|
-        ch = c.create_channel
-        q  = ch.queue("bunny.tests.recovery.client-named#{rand}")
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-        ensure_queue_recovery(ch, q)
-        q.delete
-      end
+  it "recovers passively declared exchanges and their bindings" do
+    with_open do |c|
+      ch          = c.create_channel
+      ch.confirm_select
+
+      source      = ch.fanout("amq.fanout", passive: true)
+      destination = ch.fanout("destination.exchange.recovery.example", auto_delete: true)
+
+      destination.bind(source)
+
+      # Exchanges won't get auto-deleted on connection loss unless they have
+      # had an exclusive queue bound to them.
+      dst_queue   = ch.queue("", exclusive: true)
+      dst_queue.bind(destination, routing_key: "")
+
+      src_queue   = ch.queue("", exclusive: true)
+      src_queue.bind(source, routing_key: "")
+
+      close_all_connections!
+
+      wait_for_recovery_with { connections.any? }
+
+      source.publish("msg", routing_key: "")
+      ch.wait_for_confirms
+
+      expect(dst_queue.message_count).to eq 1
+      destination.delete
     end
+  end
 
+  # this is a simplistic test that primarily execises the code path from #412
+  it "recovers exchanges that were declared with passive = true" do
+    with_open do |c|
+      ch          = c.create_channel
+      ch2         = c.create_channel
+      source      = ch.fanout("source.exchange.recovery.example", auto_delete: true)
+      destination = ch.fanout("destination.exchange.recovery.example", auto_delete: true)
 
-    it "recovers server-named queues" do
-      with_open do |c|
-        ch = c.create_channel
-        q  = ch.queue("", :exclusive => true)
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
+      source2      = ch2.fanout("source.exchange.recovery.example", no_declare: true)
 
-        wait_for_recovery
-        ch.should be_open
-        ensure_queue_recovery(ch, q)
-      end
+      destination.bind(source)
+
+      # Exchanges won't get auto-deleted on connection loss unless they have
+      # had an exclusive queue bound to them.
+      dst_queue   = ch.queue("", exclusive: true)
+      dst_queue.bind(destination, routing_key: "")
+
+      src_queue   = ch.queue("", exclusive: true)
+      src_queue.bind(source, routing_key: "")
+
+      close_all_connections!
+
+      wait_for_recovery_with { connections.any? && exchange_names_in_vhost("/").include?(source.name) }
+
+      ch2.confirm_select
+
+      source2.publish("msg", routing_key: "")
+      ch2.wait_for_confirms
+      expect(dst_queue.message_count).to eq 1
     end
+  end
 
-    it "recovers queue bindings" do
-      with_open do |c|
-        ch = c.create_channel
-        x  = ch.fanout("amq.fanout")
-        q  = ch.queue("", :exclusive => true)
-        q.bind(x)
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-        ensure_queue_binding_recovery(x, q)
-      end
+  it "recovers allocated channel ids" do
+    with_open do |c|
+      q = "queue#{Time.now.to_i}"
+      10.times { c.create_channel }
+      expect(c.queue_exists?(q)).to eq false
+      close_all_connections!
+      wait_for_recovery_with { channels.any? }
+      # make sure the connection isn't closed shortly after
+      # due to "second 'channel.open' seen". MK.
+      expect(c).to be_open
+      sleep 0.1
+      expect(c).to be_open
+      sleep 0.1
+      expect(c).to be_open
     end
+  end
 
-    it "recovers exchange bindings" do
-      with_open do |c|
-        ch = c.create_channel
-        x  = ch.fanout("amq.fanout")
-        x2 = ch.fanout("bunny.tests.recovery.fanout")
-        x2.bind(x)
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-        ensure_exchange_binding_recovery(ch, x, x2)
+  it "recovers consumers" do
+    with_open do |c|
+      delivered = false
+
+      ch = c.create_channel
+      ch.confirm_select
+      q  = ch.queue("", exclusive: true)
+      q.subscribe do |_, _, _|
+        delivered = true
       end
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+
+      q.publish("")
+      ch.wait_for_confirms
+
+      poll_until { delivered }
     end
+  end
 
-    it "recovers allocated channel ids" do
-      with_open do |c|
-        q = "queue#{Time.now.to_i}"
-        10.times { c.create_channel }
-        c.queue_exists?(q).should be_false
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        c.queue_exists?(q).should be_false
-        # make sure the connection isn't closed shortly after
-        # due to "second 'channel.open' seen". MK.
-        c.should be_open
-        sleep 0.1
-        c.should be_open
-        sleep 0.1
-        c.should be_open
-      end
+  it "recovers all consumers" do
+    n = 32
+
+    with_open do |c|
+      ch = c.create_channel
+      q  = ch.queue("", exclusive: true)
+      n.times { q.subscribe { |_, _, _| } }
+      close_all_connections!
+      wait_for_recovery_with { connections.any? }
+      expect(ch).to be_open
+      sleep 0.5
+
+      expect(q.consumer_count).to eq n
     end
+  end
+
+  it "recovers all queues" do
+    n = 32
+
+    qs = []
 
-    it "recovers consumers" do
-      with_open do |c|
-        delivered = false
-
-        ch = c.create_channel
-        q  = ch.queue("", :exclusive => true)
-        q.subscribe do |_, _, _|
-          delivered = true
-        end
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-
-        q.publish("")
-        sleep 0.5
-        expect(delivered).to be_true
+    with_open do |c|
+      ch = c.create_channel
+
+      n.times do
+        qs << ch.queue("", exclusive: true)
       end
-    end
+      close_all_connections!
+      wait_for_recovery_with { queue_names.include?(qs.first.name) }
+      sleep 0.5
+      expect(ch).to be_open
 
-    it "recovers all consumers" do
-      n = 1024
-
-      with_open do |c|
-        ch = c.create_channel
-        q  = ch.queue("", :exclusive => true)
-        n.times do
-          q.subscribe do |_, _, _|
-            delivered = true
-          end
-        end
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
-
-        wait_for_recovery
-        ch.should be_open
-
-        q.consumer_count.should == n
+      qs.each do |q|
+        ch.queue_declare(q.name, passive: true)
       end
     end
+  end
 
-    it "recovers all queues" do
-      n = 256
-
-      qs = []
+  def exchange_names_in_vhost(vhost)
+    http_client.list_exchanges(vhost).map {|e| e["name"]}
+  end
 
-      with_open do |c|
-        ch = c.create_channel
+  def connections
+    http_client.list_connections
+  end
 
-        n.times do
-          qs << ch.queue("", :exclusive => true)
-        end
-        close_all_connections!
-        sleep 0.1
-        c.should_not be_open
+  def channels
+    http_client.list_channels
+  end
 
-        wait_for_recovery
-        ch.should be_open
+  def queue_names
+    http_client.list_queues.map {|q| q["name"]}
+  end
 
-        qs.each do |q|
-          ch.queue_declare(q.name, :passive => true)
-        end
-      end
+  def close_all_connections!
+    # let whatever actions were taken before
+    # this call a chance to propagate, e.g. to make
+    # sure that connections are accounted for in the
+    # stats DB.
+    #
+    # See bin/ci/before_build for management plugin
+    # pre-configuration.
+    #
+    # MK.
+    sleep 1.1
+    connections.each do |conn_info|
+      close_ignoring_permitted_exceptions(conn_info.name)
     end
   end
+
+  def close_ignoring_permitted_exceptions(connection_name)
+    http_client.close_connection(connection_name)
+  rescue Bunny::ConnectionForced, Faraday::ResourceNotFound
+    # ignored
+  end
+
+  def wait_for_recovery_with(&probe)
+    poll_until &probe
+  end
+
+  def poll_while(&probe)
+    Bunny::TestKit.poll_while(&probe)
+  end
+
+  def poll_until(&probe)
+    Bunny::TestKit.poll_until(&probe)
+  end
+
+  def with_open(c = Bunny.new(network_recovery_interval: recovery_interval,
+                              recover_from_connection_close: true,
+                              logger: logger), &block)
+    c.start
+    block.call(c)
+  ensure
+    c.close(false) rescue nil
+  end
+
+  def with_open_multi_host(&block)
+    c = Bunny.new(hosts: ["127.0.0.1", "localhost"],
+                  network_recovery_interval: recovery_interval,
+                  recover_from_connection_close: true,
+                  logger: logger)
+    with_open(c, &block)
+  end
+
+  def with_open_multi_broken_host(&block)
+    c = Bunny.new(hosts: ["broken", "127.0.0.1", "localhost"],
+                  hosts_shuffle_strategy: Proc.new { |hosts| hosts }, # We do not shuffle for these tests so we always hit the broken host
+                  network_recovery_interval: recovery_interval,
+                  recover_from_connection_close: true,
+                  logger: logger)
+    with_open(c, &block)
+  end
+
+  def ensure_queue_recovery(ch, q)
+    ch.confirm_select
+    q.purge
+    x = ch.default_exchange
+    x.publish("msg", routing_key: q.name)
+    ch.wait_for_confirms
+    expect(q.message_count).to eq 1
+    q.purge
+  end
+
+  def ensure_queue_binding_recovery(ch, x, q, routing_key = "")
+    ch.confirm_select
+    q.purge
+    x.publish("msg", routing_key: routing_key)
+    ch.wait_for_confirms
+    expect(q.message_count).to eq 1
+    q.purge
+  end
 end