hutch 0.25.0.pre.rc1-java → 2.0.0.rc1-java

data/spec/hutch/worker_spec.rb

@@ -4,7 +4,8 @@ require 'hutch/worker'
 describe Hutch::Worker do
   let(:consumer) { double('Consumer', routing_keys: %w( a b c ),
                           get_queue_name: 'consumer', get_arguments: {},
-                          get_serializer: nil) }
+                          get_options: {}, get_serializer: nil,
+                          without_namespace?: false) }
   let(:consumers) { [consumer, double('Consumer')] }
   let(:broker) { Hutch::Broker.new }
   let(:setup_procs) { Array.new(2) { Proc.new {} } }
@@ -32,20 +33,34 @@ describe Hutch::Worker do
 
   describe '#setup_queue' do
     let(:queue) { double('Queue', bind: nil, subscribe: nil) }
-    before { allow(broker).to receive_messages(queue: queue, bind_queue: nil) }
+    before { allow(broker).to receive_messages(queue: queue, bind_queue: nil, namespaced_queue_name: 'consumer') }
 
-    it 'creates a queue' do
-      expect(broker).to receive(:queue).with(consumer.get_queue_name, consumer.get_arguments).and_return(queue)
+    it 'creates a queue with a namespaced name' do
+      expect(broker).to receive(:namespaced_queue_name).with('consumer').and_return('ns:consumer')
+      expect(broker).to receive(:queue).with('ns:consumer', consumer.get_options).and_return(queue)
       worker.setup_queue(consumer)
     end
 
+    context 'when the consumer uses without_namespace' do
+      let(:consumer) { double('Consumer', routing_keys: %w( a b c ),
+                              get_queue_name: 'consumer', get_arguments: {},
+                              get_options: {}, get_serializer: nil,
+                              without_namespace?: true) }
+
+      it 'creates a queue without applying the namespace' do
+        expect(broker).not_to receive(:namespaced_queue_name)
+        expect(broker).to receive(:queue).with('consumer', consumer.get_options).and_return(queue)
+        worker.setup_queue(consumer)
+      end
+    end
+
     it 'binds the queue to each of the routing keys' do
       expect(broker).to receive(:bind_queue).with(queue, %w( a b c ))
       worker.setup_queue(consumer)
     end
 
     it 'sets up a subscription' do
-      expect(queue).to receive(:subscribe).with(consumer_tag: %r(^hutch\-.{36}$), manual_ack: true)
+      expect(queue).to receive(:subscribe).with(consumer_tag: %r(^hutch\-.{36}$), manual_ack: true, on_cancellation: kind_of(Proc))
       worker.setup_queue(consumer)
     end
 
@@ -53,7 +68,7 @@ describe Hutch::Worker do
       before { Hutch::Config.set(:consumer_tag_prefix, 'appname') }
 
       it 'sets up a subscription with the configured tag prefix' do
-        expect(queue).to receive(:subscribe).with(consumer_tag: %r(^appname\-.{36}$), manual_ack: true)
+        expect(queue).to receive(:subscribe).with(consumer_tag: %r(^appname\-.{36}$), manual_ack: true, on_cancellation: kind_of(Proc))
         worker.setup_queue(consumer)
       end
     end
@@ -61,6 +76,7 @@ describe Hutch::Worker do
     context 'with a configured consumer tag prefix that is too long' do
       let(:maximum_size) { 255 - SecureRandom.uuid.size - 1 }
       before { Hutch::Config.set(:consumer_tag_prefix, 'a'.*(maximum_size + 1)) }
+      after { Hutch::Config.set(:consumer_tag_prefix, 'hutch') }
 
       it 'raises an error' do
         expect { worker.setup_queue(consumer) }.to raise_error(/Tag must be 255 bytes long at most/)
@@ -68,28 +84,87 @@ describe Hutch::Worker do
     end
   end
 
+  describe '#handle_cancellation' do
+    let(:cancelled_channel) { double('Channel') }
+    let(:new_channel) { double('Channel') }
+    let(:log) { StringIO.new }
+
+    before do
+      allow(Hutch::Logging).to receive(:logger).and_return(Logger.new(log))
+      allow(broker).to receive(:channel).and_return(cancelled_channel)
+      allow(broker).to receive(:replace_channel!) do
+        allow(broker).to receive(:channel).and_return(new_channel)
+      end
+      allow(worker).to receive(:setup_queues)
+    end
+
+    context 'when the queue still exists' do
+      before { allow(broker).to receive(:queue_exists?).with('consumer').and_return(true) }
+
+      it 'replaces the channel and re-subscribes' do
+        expect(broker).to receive(:replace_channel!)
+        expect(worker).to receive(:setup_queues)
+
+        worker.handle_cancellation('consumer', cancelled_channel).join
+      end
+
+      it 'replaces the channel once for all the consumers that shared it' do
+        expect(broker).to receive(:replace_channel!).once
+
+        [
+          worker.handle_cancellation('consumer', cancelled_channel),
+          worker.handle_cancellation('consumer', cancelled_channel)
+        ].each(&:join)
+      end
+
+      it 'does not replace a channel that has already been replaced' do
+        expect(broker).not_to receive(:replace_channel!)
+
+        worker.handle_cancellation('consumer', double('An older channel')).join
+      end
+    end
+
+    context 'when the queue is gone' do
+      before { allow(broker).to receive(:queue_exists?).with('consumer').and_return(false) }
+
+      it 'does not re-subscribe' do
+        expect(broker).not_to receive(:replace_channel!)
+
+        worker.handle_cancellation('consumer', cancelled_channel)
+
+        log.rewind
+        expect(log.read).to match(/the queue no longer exists/)
+      end
+    end
+  end
+
   describe '#handle_message' do
+    subject { worker.handle_message(consumer, delivery_info, properties, payload) }
     let(:payload) { '{}' }
     let(:consumer_instance) { double('Consumer instance') }
-    let(:delivery_info) { double('Delivery Info', routing_key: '',
+    let(:delivery_info) { double('Delivery Info', routing_key: '', channel: nil,
                                  delivery_tag: 'dt') }
     let(:properties) { double('Properties', message_id: nil, content_type: "application/json") }
+    let(:log) { StringIO.new }
     before { allow(consumer).to receive_messages(new: consumer_instance) }
     before { allow(broker).to receive(:ack) }
     before { allow(broker).to receive(:nack) }
     before { allow(consumer_instance).to receive(:broker=) }
     before { allow(consumer_instance).to receive(:delivery_info=) }
+    before { allow(Hutch::Logging).to receive(:logger).and_return(Logger.new(log)) }
 
     it 'passes the message to the consumer' do
       expect(consumer_instance).to receive(:process).
                         with(an_instance_of(Hutch::Message))
-      worker.handle_message(consumer, delivery_info, properties, payload)
+      expect(consumer_instance).to receive(:message_rejected?).and_return(false)
+      subject
     end
 
     it 'acknowledges the message' do
       allow(consumer_instance).to receive(:process)
-      expect(broker).to receive(:ack).with(delivery_info.delivery_tag)
-      worker.handle_message(consumer, delivery_info, properties, payload)
+      expect(broker).to receive(:ack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
+      expect(consumer_instance).to receive(:message_rejected?).and_return(false)
+      subject
     end
 
     context 'when the consumer fails and a requeue is configured' do
@@ -97,7 +172,7 @@ describe Hutch::Worker do
       it 'requeues the message' do
         allow(consumer_instance).to receive(:process).and_raise('failed')
         requeuer = double
-        allow(requeuer).to receive(:handle).ordered { |delivery_info, properties, broker, e|
+        allow(requeuer).to receive(:handle) { |delivery_info, properties, broker, e|
           broker.requeue delivery_info.delivery_tag
           true
         }
@@ -106,47 +181,86 @@ describe Hutch::Worker do
         expect(broker).to_not receive(:nack)
         expect(broker).to receive(:requeue)
 
-        worker.handle_message(consumer, delivery_info, properties, payload)
+        subject
       end
     end
 
 
     context 'when the consumer raises an exception' do
+      let(:expected_log) { /ERROR .+ error in consumer .+ RuntimeError .+ backtrace:/m }
       before { allow(consumer_instance).to receive(:process).and_raise('a consumer error') }
 
       it 'logs the error' do
-        Hutch::Config[:error_handlers].each do |backend|
-          expect(backend).to receive(:handle)
-        end
-        worker.handle_message(consumer, delivery_info, properties, payload)
+        expect { subject }.to change { log.tap(&:rewind).read }.from("").to(expected_log)
       end
 
       it 'rejects the message' do
-        expect(broker).to receive(:nack).with(delivery_info.delivery_tag)
-        worker.handle_message(consumer, delivery_info, properties, payload)
+        expect(broker).to receive(:nack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
+        subject
+      end
+
+      context 'when a custom error handler supports delivery info' do
+        let(:error_handler) do
+          Class.new(Hutch::ErrorHandlers::Base) do
+            def handle(_properties, _payload, _consumer, _ex, delivery_info)
+              raise unless delivery_info.delivery_tag == 'dt'
+              puts 'handled!'
+            end
+          end
+        end
+
+        around do |example|
+          original = Hutch::Config[:error_handlers]
+          Hutch::Config[:error_handlers] = [error_handler.new]
+          example.run
+          Hutch::Config[:error_handlers] = original
+        end
+
+        it 'calls the custom handler with delivery info' do
+          expect { subject }.to output("handled!\n").to_stdout
+        end
+      end
+
+      context 'when a custom error handler does not support delivery info' do
+        let(:error_handler) do
+          Class.new(Hutch::ErrorHandlers::Base) do
+            def handle(_properties, _payload, _consumer, _ex)
+              puts 'handled!'
+            end
+          end
+        end
+
+        around do |example|
+          original = Hutch::Config[:error_handlers]
+          Hutch::Config[:error_handlers] = [error_handler.new]
+          example.run
+          Hutch::Config[:error_handlers] = original
+        end
+
+        it 'calls the custom handler with delivery info' do
+          expect { subject }.to output("handled!\n").to_stdout
+        end
       end
     end
 
     context "when the payload is not valid json" do
       let(:payload) { "Not Valid JSON" }
+      let(:expected_log) { /ERROR .+ error in consumer .+ JSON::ParserError .+ backtrace:/m }
 
       it 'logs the error' do
-        Hutch::Config[:error_handlers].each do |backend|
-          expect(backend).to receive(:handle)
-        end
-        worker.handle_message(consumer, delivery_info, properties, payload)
+        expect { subject }.to change { log.tap(&:rewind).read }.from("").to(expected_log)
       end
 
       it 'rejects the message' do
-        expect(broker).to receive(:nack).with(delivery_info.delivery_tag)
-        worker.handle_message(consumer, delivery_info, properties, payload)
+        expect(broker).to receive(:nack).with(delivery_info.delivery_tag, channel: delivery_info.channel)
+        subject
       end
     end
   end
 
 
   describe '#acknowledge_error' do
-    let(:delivery_info) { double('Delivery Info', routing_key: '',
+    let(:delivery_info) { double('Delivery Info', routing_key: '', channel: nil,
                                  delivery_tag: 'dt') }
     let(:properties) { double('Properties', message_id: 'abc123') }
 

data/spec/integration/channel_recovery_spec.rb

@@ -0,0 +1,187 @@
+require 'spec_helper'
+
+# Channel recovery is bunny-only, see MarchHareAdapter#install_channel_recovery
+return if defined?(JRUBY_VERSION)
+require 'hutch/broker'
+require 'hutch/worker'
+require 'hutch/consumer'
+require 'bunny'
+require 'json'
+require 'securerandom'
+
+describe 'channel recovery after delivery acknowledgement timeout', rabbitmq: true, adapter: :bunny do
+  CONSUMER_TIMEOUT_MS = 5_000
+  # A 4.3 quorum queue times the consumer out on its own timer, which fires at
+  # the deadline. Earlier series detect it from the channel tick instead, and
+  # `channel_tick_interval` defaults to 60s, so detection lags by up to that much.
+  BLOCKED_HANDLER_SECONDS = 20
+  BLOCKED_HANDLER_SECONDS_BEFORE_4_3 = 100
+
+  let(:log) { StringIO.new }
+  let(:logger) { Logger.new(log) }
+  let(:exchange_name) { "hutch.integration.exchange.#{SecureRandom.hex(4)}" }
+  let(:queue_name) { "hutch.integration.queue.#{SecureRandom.hex(4)}" }
+  let(:routing_key) { "hutch.integration.key.#{SecureRandom.hex(4)}" }
+
+  let(:processed) { [] }
+  let(:processed_lock) { Mutex.new }
+  let(:timed_out_once) { [false] }
+
+  let(:server_version) do
+    Gem::Version.new(publisher.server_properties['version'].to_s[/\A\d+(\.\d+)*/])
+  end
+
+  let(:blocked_handler_seconds) do
+    if server_version >= Gem::Version.new('4.3')
+      BLOCKED_HANDLER_SECONDS
+    else
+      BLOCKED_HANDLER_SECONDS_BEFORE_4_3
+    end
+  end
+
+  let(:consumer_class) do
+    msgs = processed
+    lock = processed_lock
+    rk = routing_key
+    qn = queue_name
+    timed_out = timed_out_once
+    blocked = blocked_handler_seconds
+
+    Class.new do
+      include Hutch::Consumer
+
+      consume rk
+      queue_name qn
+      arguments(
+        'x-queue-type' => 'quorum',
+        'x-consumer-timeout' => CONSUMER_TIMEOUT_MS
+      )
+
+      # Both `CONSUMER_TIMEOUT_MS` and this sleep run from the same delivery,
+      # so the margin between them does not shrink on a loaded machine.
+      define_method(:process) do |message|
+        if message['id'] == 'trigger-timeout' && !timed_out[0]
+          timed_out[0] = true
+          sleep blocked
+        end
+
+        lock.synchronize { msgs << message['id'] }
+      end
+    end
+  end
+
+  let(:broker) { Hutch::Broker.new }
+  let(:worker) { Hutch::Worker.new(broker, [consumer_class], []) }
+
+  let(:publisher) do
+    Bunny.new(
+      host: Hutch::Config[:mq_host],
+      port: Hutch::Config[:mq_port],
+      username: Hutch::Config[:mq_username],
+      password: Hutch::Config[:mq_password],
+      vhost: Hutch::Config[:mq_vhost]
+    ).tap(&:start)
+  end
+
+  let(:publisher_channel) { publisher.create_channel }
+  let(:exchange) { publisher_channel.topic(exchange_name, durable: true) }
+
+  before do
+    Hutch::Logging.logger = logger
+    Hutch::Config.set(:mq_exchange, exchange_name)
+    Hutch::Config.set(:force_publisher_confirms, false)
+    Hutch::Config.set(:client_logger, logger)
+    # Channel replacement waits this long for the deliberately blocked
+    # handler: the default of 11s would use up most of the consumption window.
+    @graceful_exit_timeout = Hutch::Config[:graceful_exit_timeout]
+    Hutch::Config.set(:graceful_exit_timeout, 1)
+  end
+
+  after do
+    publisher_channel.close rescue nil
+    publisher.close rescue nil
+    broker.disconnect rescue nil
+    Hutch::Config.set(:graceful_exit_timeout, @graceful_exit_timeout)
+    Hutch::Logging.logger = Logger.new(File::NULL)
+  end
+
+  def wait_for(timeout, label, &condition)
+    await_condition(timeout, label, &condition)
+  rescue AwaitHelpers::ConditionTimeout => e
+    raise <<~MSG
+      #{e.message}
+
+      processed_messages=#{processed_messages.inspect}
+      channel_open=#{broker.channel.open? rescue 'unknown'}
+      channel_closed=#{broker.channel.closed? rescue 'unknown'}
+
+      log output:
+      #{log_output}
+    MSG
+  end
+
+  def processed_messages
+    processed_lock.synchronize { processed.dup }
+  end
+
+  # `StringIO#string` does not touch the stream position: a rewind here would
+  # race with the worker threads that log concurrently and corrupt the buffer.
+  def log_output
+    log.string
+  end
+
+  def publish_message(id)
+    exchange.publish(
+      JSON.dump('id' => id),
+      routing_key: routing_key,
+      content_type: 'application/json',
+      persistent: true
+    )
+  end
+
+  # RabbitMQ up to 4.2 closes the channel on a delivery acknowledgement
+  # timeout; 4.3+ quorum queues instead cancel only the timed out consumer.
+  let(:cancels_the_consumer) { server_version >= Gem::Version.new('4.3') }
+
+  let(:recovery_log_pattern) do
+    if cancels_the_consumer
+      /cancelled by the server, re-subscribing/i
+    else
+      /delivery acknowledgement on channel \d+ timed out/i
+    end
+  end
+
+  # Replacing the channel requeues what the cancelled consumer still held, so
+  # consumption resumes at once. Reopening one does not, so it resumes at the
+  # next consumer timeout tick.
+  let(:recovery_seconds) { cancels_the_consumer ? 15 : 90 }
+
+  # This spec is intentionally slow because RabbitMQ enforces delivery
+  # acknowledgement timeouts periodically on a timer, not immediately at the deadline.
+  it 're-subscribes and consumes later messages after RabbitMQ times out an unacknowledged delivery' do
+    broker.connect
+    worker.setup_queues
+
+    publish_message('trigger-timeout')
+
+    wait_for(240, 'a delivery acknowledgement timeout to close the channel or cancel the consumer') do
+      log_output.match?(recovery_log_pattern)
+    end
+
+    publish_message('after-recovery')
+
+    wait_for(recovery_seconds, 'after-recovery message consumption') do
+      processed_messages.include?('after-recovery')
+    end
+
+    expect(log_output).to match(recovery_log_pattern)
+
+    # A consumed message proves the recovery finished, the publisher rebuild
+    # included.
+    broker.publish(routing_key, 'id' => 'published-after-recovery')
+
+    wait_for(15, 'published-after-recovery message consumption') do
+      processed_messages.include?('published-after-recovery')
+    end
+  end
+end

data/spec/integration/publish_consume_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+require 'hutch/broker'
+require 'hutch/worker'
+require 'hutch/consumer'
+require 'securerandom'
+
+describe 'publishing and consuming messages', rabbitmq: true, adapter: :bunny do
+  let(:exchange_name) { "hutch.test.#{SecureRandom.hex(4)}" }
+  let(:routing_key) { "test.message" }
+  let(:received) { [] }
+
+  let(:consumer_class) do
+    msgs = received
+    rk = routing_key
+    qn = "test_consumer_#{SecureRandom.hex(4)}"
+
+    Class.new do
+      include Hutch::Consumer
+      consume rk
+      queue_name qn
+
+      define_method(:process) { |message| msgs << message.body }
+    end
+  end
+
+  let(:broker) { Hutch::Broker.new }
+  let(:worker) { Hutch::Worker.new(broker, [consumer_class], []) }
+
+  before do
+    Hutch::Config.set(:mq_exchange, exchange_name)
+  end
+
+  after do
+    broker.disconnect rescue nil
+  end
+
+  it 'publishes and consumes a message' do
+    broker.connect
+    worker.setup_queues
+
+    broker.publish(routing_key, { test: 'data' })
+
+    await_condition(15, 'message consumption') { received.any? }
+
+    expect(received.first).to eq('test' => 'data')
+  end
+end