hutch 0.21.0-java → 0.25.0-java

data/lib/hutch/worker.rb

@@ -2,17 +2,18 @@ require 'hutch/message'
 require 'hutch/logging'
 require 'hutch/broker'
 require 'hutch/acknowledgements/nack_on_all_failures'
+require 'hutch/waiter'
 require 'carrot-top'
+require 'securerandom'
 
 module Hutch
   class Worker
     include Logging
 
-    SHUTDOWN_SIGNALS = %w(QUIT TERM INT)
-
-    def initialize(broker, consumers)
+    def initialize(broker, consumers, setup_procs)
       @broker        = broker
       self.consumers = consumers
+      self.setup_procs = setup_procs
     end
 
     # Run the main event loop. The consumers will be set up with queues, and
@@ -20,46 +21,11 @@ module Hutch
     # never returns.
     def run
       setup_queues
+      setup_procs.each(&:call)
 
-      # Set up signal handlers for graceful shutdown
-      register_signal_handlers
-
-      main_loop
-    end
-
-    def main_loop
-      if defined?(JRUBY_VERSION)
-        # Binds shutdown listener to notify main thread if channel was closed
-        bind_shutdown_handler
-
-        handle_signals until shutdown_not_called?(0.1)
-      else
-        # Take a break from Thread#join every 0.1 seconds to check if we've
-        # been sent any signals
-        handle_signals until @broker.wait_on_threads(0.1)
-      end
-    end
-
-    # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
-    # gracefully. Forceful shutdowns are very bad!
-    def register_signal_handlers
-      Thread.main[:signal_queue] = []
-      supported_shutdown_signals.each do |sig|
-        # This needs to be reentrant, so we queue up signals to be handled
-        # in the run loop, rather than acting on signals here
-        trap(sig) do
-          Thread.main[:signal_queue] << sig
-        end
-      end
-    end
+      Waiter.wait_until_signaled
 
-    # Handle any pending signals
-    def handle_signals
-      signal = Thread.main[:signal_queue].shift
-      if signal
-        logger.info "caught sig#{signal.downcase}, stopping hutch..."
-        stop
-      end
+      stop
     end
 
     # Stop a running worker by killing all subscriber threads.
@@ -67,36 +33,24 @@ module Hutch
       @broker.stop
     end
 
-    # Binds shutdown handler, called if channel is closed or network Failed
-    def bind_shutdown_handler
-      @broker.channel.on_shutdown do
-        Thread.main[:shutdown_received] = true
-      end
-    end
-
-    # Checks if shutdown handler was called, then sleeps for interval
-    def shutdown_not_called?(interval)
-      if Thread.main[:shutdown_received]
-        true
-      else
-        sleep(interval)
-        false
-      end
-    end
-
     # Set up the queues for each of the worker's consumers.
     def setup_queues
       logger.info 'setting up queues'
-      @consumers.each { |consumer| setup_queue(consumer) }
+      vetted = @consumers.reject { |c| group_configured? && group_restricted?(c) }
+      vetted.each do |c|
+        setup_queue(c)
+      end
     end
 
     # Bind a consumer's routing keys to its queue, and set up a subscription to
     # receive messages sent to the queue.
     def setup_queue(consumer)
+      logger.info "setting up queue: #{consumer.get_queue_name}"
+
       queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
       @broker.bind_queue(queue, consumer.routing_keys)
 
-      queue.subscribe(manual_ack: true) do |*args|
+      queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
         delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
         handle_message(consumer, delivery_info, properties, payload)
       end
@@ -106,7 +60,7 @@ module Hutch
     # for wrapping up the message and passing it to the consumer.
     def handle_message(consumer, delivery_info, properties, payload)
       serializer = consumer.get_serializer || Hutch::Config[:serializer]
-      logger.info {
+      logger.debug {
         spec   = serializer.binary? ? "#{payload.bytesize} bytes" : "#{payload}"
         "message(#{properties.message_id || '-'}): " +
         "routing key: #{delivery_info.routing_key}, " +
@@ -120,16 +74,16 @@ module Hutch
       @broker.ack(delivery_info.delivery_tag)
     rescue => ex
       acknowledge_error(delivery_info, properties, @broker, ex)
-      handle_error(properties.message_id, payload, consumer, ex)
+      handle_error(properties, payload, consumer, ex)
     end
 
     def with_tracing(klass)
       Hutch::Config[:tracer].new(klass)
     end
 
-    def handle_error(message_id, payload, consumer, ex)
+    def handle_error(*args)
       Hutch::Config[:error_handlers].each do |backend|
-        backend.handle(message_id, payload, consumer, ex)
+        backend.handle(*args)
       end
     end
 
@@ -154,8 +108,38 @@ module Hutch
 
     private
 
-    def supported_shutdown_signals
-      SHUTDOWN_SIGNALS.keep_if { |s| Signal.list.keys.include? s }.map(&:to_sym)
+    def group_configured?
+      if group.present? && consumer_groups.blank?
+        logger.info 'Consumer groups are blank'
+      end
+      group.present?
+    end
+
+    def group_restricted?(consumer)
+      consumers_to_load = consumer_groups[group]
+      if consumers_to_load
+        !allowed_consumers.include?(consumer.name)
+      else
+        true
+      end
+    end
+
+    def group
+      Hutch::Config[:group]
+    end
+
+    def consumer_groups
+      Hutch::Config[:consumer_groups]
+    end
+
+    attr_accessor :setup_procs
+
+    def unique_consumer_tag
+      prefix = Hutch::Config[:consumer_tag_prefix]
+      unique_part = SecureRandom.uuid
+      "#{prefix}-#{unique_part}".tap do |tag|
+        raise "Tag must be 255 bytes long at most, current one is #{tag.bytesize} ('#{tag}')" if tag.bytesize > 255
+      end
     end
   end
 end

data/lib/yard-settings/handler.rb

@@ -0,0 +1,38 @@
+# :nodoc:
+class SettingsHandlerBase < YARD::Handlers::Ruby::Base
+  handles method_call :string_setting
+  handles method_call :number_setting
+  handles method_call :boolean_setting
+
+  namespace_only
+
+  def process
+    name = statement.parameters.first.jump(:tstring_content, :ident).source
+    object = YARD::CodeObjects::MethodObject.new(namespace, name)
+    register(object)
+
+    # Modify the code object for the new instance method
+    object.dynamic = true
+    # Add custom metadata to the object
+    object['custom_field'] = '(Found using method_missing)'
+
+    # Module-level configuration notes
+    hutch_config = YARD::CodeObjects::ModuleObject.new(:root, "Hutch::Config")
+    collection_name = statement.first.first
+    default_value = statement.parameters[1].jump(:tstring_content, :ident).source
+
+    (hutch_config['setting_rows'] ||= []) << {
+      name: name,
+      default_value: default_value,
+      type: collection_name.sub('_setting', '').capitalize,
+      description: object.docstring,
+      first_line_of_description: first_line_of_description(object)
+    }
+  end
+
+  def first_line_of_description(object)
+    return '' if object.docstring.blank?
+
+    object.docstring.lines.first
+  end
+end

data/lib/yard-settings/yard-settings.rb

@@ -0,0 +1,2 @@
+YARD::Templates::Engine.register_template_path(File.dirname(__FILE__) + '/../../templates')
+require File.join(File.dirname(__FILE__), 'handler') if RUBY19

data/spec/hutch/broker_spec.rb

@@ -2,8 +2,16 @@ require 'spec_helper'
 require 'hutch/broker'
 
 describe Hutch::Broker do
-  let(:config) { deep_copy(Hutch::Config.user_config) }
-  subject(:broker) { Hutch::Broker.new(config) }
+  before do
+    Hutch::Config.initialize(client_logger: Hutch::Logging.logger)
+    @config = Hutch::Config.to_hash
+  end
+  let!(:config) { @config }
+  after do
+    Hutch::Config.instance_variable_set(:@config, nil)
+    Hutch::Config.initialize
+  end
+  let(:broker) { Hutch::Broker.new(config) }
 
   describe '#connect' do
     before { allow(broker).to receive(:set_up_amqp_connection) }
@@ -53,93 +61,183 @@ describe Hutch::Broker do
     end
   end
 
-  describe '#set_up_amqp_connection', rabbitmq: true do
-    context 'with valid details' do
-      before { broker.set_up_amqp_connection }
-      after  { broker.disconnect }
+  describe '#set_up_amqp_connection' do
+    it 'opens a connection, channel and declares an exchange' do
+      expect(broker).to receive(:open_connection!).ordered
+      expect(broker).to receive(:open_channel!).ordered
+      expect(broker).to receive(:declare_exchange!).ordered
 
-      describe '#connection', adapter: :bunny do
-        subject { super().connection }
-        it { is_expected.to be_a Hutch::Adapters::BunnyAdapter }
-      end
+      broker.set_up_amqp_connection
+    end
+  end
 
-      describe '#connection', adapter: :march_hare do
-        subject { super().connection }
-        it { is_expected.to be_a Hutch::Adapters::MarchHareAdapter }
-      end
+  describe '#open_connection', rabbitmq: true do
+    describe 'return value' do
+      subject { broker.open_connection }
+      after { subject.close }
 
-      describe '#channel', adapter: :bunny do
-        subject { super().channel }
-        it { is_expected.to be_a Bunny::Channel }
-      end
+      it(nil, adapter: :bunny)      { is_expected.to be_a Hutch::Adapters::BunnyAdapter }
+      it(nil, adapter: :march_hare) { is_expected.to be_a Hutch::Adapters::MarchHareAdapter }
+    end
 
-      describe '#channel', adapter: :march_hare do
-        subject { super().channel }
-        it { is_expected.to be_a MarchHare::Channel }
+    context 'when given invalid details' do
+      before { config[:mq_host] = 'notarealhost' }
+      it { expect { broker.open_connection }.to raise_error(StandardError) }
+    end
+
+    it 'does not set #connection' do
+      connection = broker.open_connection
+
+      expect(broker.connection).to be_nil
+
+      connection.close
+    end
+
+    context 'when configured with a URI' do
+      context 'which specifies the port' do
+        before { config[:uri] = 'amqp://guest:guest@127.0.0.1:5672/' }
+
+        it 'successfully connects' do
+          c = broker.open_connection
+          expect(c).to be_open
+          c.close
+        end
       end
 
-      describe '#exchange', adapter: :bunny do
-        subject { super().exchange }
-        it { is_expected.to be_a Bunny::Exchange }
+      context 'which does not specify port and uses the amqp scheme' do
+        before { config[:uri] = 'amqp://guest:guest@127.0.0.1/' }
+
+        it 'successfully connects' do
+          c = broker.open_connection
+          expect(c).to be_open
+          c.close
+        end
       end
 
-      describe '#exchange', adapter: :march_hare do
-        subject { super().exchange }
-        it { is_expected.to be_a MarchHare::Exchange }
+      context 'which specifies the amqps scheme' do
+        before { config[:uri] = 'amqps://guest:guest@127.0.0.1/' }
+
+        it 'utilises TLS' do
+          expect(Hutch::Adapter).to receive(:new).with(
+            hash_including(tls: true, port: 5671)
+          ).and_return(instance_double('Hutch::Adapter', start: nil))
+
+          broker.open_connection
+        end
       end
     end
+  end
 
-    context 'when given invalid details' do
-      before { config[:mq_host] = 'notarealhost' }
-      let(:set_up_amqp_connection) { ->{ broker.set_up_amqp_connection } }
+  describe '#open_connection!' do
+    it 'sets the #connection to #open_connection' do
+      connection = double('connection').as_null_object
+
+      expect(broker).to receive(:open_connection).and_return(connection)
+
+      broker.open_connection!
 
-      specify { expect(set_up_amqp_connection).to raise_error }
+      expect(broker.connection).to eq(connection)
+    end
+  end
+
+  describe '#open_channel', rabbitmq: true do
+    before { broker.open_connection! }
+    after { broker.disconnect }
+
+    describe 'return value' do
+      subject { broker.open_channel }
+
+      it(nil, adapter: :bunny)      { is_expected.to be_a Bunny::Channel }
+      it(nil, adapter: :march_hare) { is_expected.to be_a MarchHare::Channel }
+    end
+
+    it 'does not set #channel' do
+      broker.open_channel
+      expect(broker.channel).to be_nil
     end
 
     context 'with channel_prefetch set' do
       let(:prefetch_value) { 1 }
       before { config[:channel_prefetch] = prefetch_value }
-      after  { broker.disconnect }
 
       it "set's channel's prefetch", adapter: :bunny do
-        expect_any_instance_of(Bunny::Channel).
-          to receive(:prefetch).with(prefetch_value)
-        broker.set_up_amqp_connection
+        expect_any_instance_of(Bunny::Channel).to receive(:prefetch).with(prefetch_value)
+        broker.open_channel
       end
 
       it "set's channel's prefetch", adapter: :march_hare do
-        expect_any_instance_of(MarchHare::Channel).
-          to receive(:prefetch=).with(prefetch_value)
-        broker.set_up_amqp_connection
+        expect_any_instance_of(MarchHare::Channel).to receive(:prefetch=).with(prefetch_value)
+        broker.open_channel
       end
     end
 
     context 'with force_publisher_confirms set' do
       let(:force_publisher_confirms_value) { true }
       before { config[:force_publisher_confirms] = force_publisher_confirms_value }
-      after  { broker.disconnect }
 
       it 'waits for confirmation', adapter: :bunny do
-        expect_any_instance_of(Bunny::Channel).
-          to receive(:confirm_select)
-        broker.set_up_amqp_connection
+        expect_any_instance_of(Bunny::Channel).to receive(:confirm_select)
+        broker.open_channel
       end
 
       it 'waits for confirmation', adapter: :march_hare do
-        expect_any_instance_of(MarchHare::Channel).
-          to receive(:confirm_select)
-        broker.set_up_amqp_connection
+        expect_any_instance_of(MarchHare::Channel).to receive(:confirm_select)
+        broker.open_channel
       end
     end
   end
 
+  describe '#open_channel!' do
+    it 'sets the #channel to #open_channel' do
+      channel = double('channel').as_null_object
+
+      expect(broker).to receive(:open_channel).and_return(channel)
+
+      broker.open_channel!
+
+      expect(broker.channel).to eq(channel)
+    end
+  end
+
+  describe '#declare_exchange' do
+    before do
+      broker.open_connection!
+      broker.open_channel!
+    end
+    after { broker.disconnect }
+
+    describe 'return value' do
+      subject { broker.declare_exchange }
+
+      it(nil, adapter: :bunny)      { is_expected.to be_a Bunny::Exchange }
+      it(nil, adapter: :march_hare) { is_expected.to be_a MarchHare::Exchange }
+    end
+
+    it 'does not set #exchange' do
+      broker.declare_exchange
+      expect(broker.exchange).to be_nil
+    end
+  end
+
+  describe '#declare_exchange!' do
+    it 'sets the #exchange to #declare_exchange' do
+      exchange = double('exchange').as_null_object
+
+      expect(broker).to receive(:declare_exchange).and_return(exchange)
+
+      broker.declare_exchange!
+
+      expect(broker.exchange).to eq(exchange)
+    end
+  end
+
   describe '#set_up_api_connection', rabbitmq: true do
     context 'with valid details' do
       before { broker.set_up_api_connection }
       after  { broker.disconnect }
 
       describe '#api_client' do
-        subject { super().api_client }
+        subject { broker.api_client }
         it { is_expected.to be_a CarrotTop }
       end
     end
@@ -149,7 +247,7 @@ describe Hutch::Broker do
       after  { broker.disconnect }
       let(:set_up_api_connection) { ->{ broker.set_up_api_connection } }
 
-      specify { expect(set_up_api_connection).to raise_error }
+      specify { expect(set_up_api_connection).to raise_error(StandardError) }
     end
   end
 
@@ -192,7 +290,7 @@ describe Hutch::Broker do
 
   describe '#bind_queue' do
 
-    around { |example| broker.connect { example.run } }
+    around { |example| broker.connect(host: "127.0.0.1") { example.run } }
 
     let(:routing_keys) { %w( a b c ) }
     let(:queue) { double('Queue', bind: nil, unbind: nil, name: 'consumer') }
@@ -225,21 +323,6 @@ describe Hutch::Broker do
     end
   end
 
-  describe '#wait_on_threads' do
-    let(:thread) { double('Thread') }
-    before { allow(broker).to receive(:work_pool_threads).and_return(threads) }
-
-    context 'when all threads finish within the timeout' do
-      let(:threads) { [double(join: thread), double(join: thread)] }
-      specify { expect(broker.wait_on_threads(1)).to be_truthy }
-    end
-
-    context 'when timeout expires for one thread' do
-      let(:threads) { [double(join: thread), double(join: nil)] }
-      specify { expect(broker.wait_on_threads(1)).to be_falsey }
-    end
-  end
-
   describe '#stop', adapter: :bunny do
     let(:thread_1) { double('Thread') }
     let(:thread_2) { double('Thread') }
@@ -280,12 +363,12 @@ describe Hutch::Broker do
 
       it 'publishes to the exchange' do
         expect(broker.exchange).to receive(:publish).once
-        broker.publish('test.key', 'message')
+        broker.publish('test.key', {key: "value"})
       end
 
       it 'sets default properties' do
         expect(broker.exchange).to receive(:publish).with(
-          JSON.dump("message"),
+          JSON.dump({key: "value"}),
           hash_including(
             persistent: true,
             routing_key: 'test.key',
@@ -293,12 +376,12 @@ describe Hutch::Broker do
           )
         )
 
-        broker.publish('test.key', 'message')
+        broker.publish('test.key', {key: "value"})
       end
 
       it 'allows passing message properties' do
         expect(broker.exchange).to receive(:publish).once
-        broker.publish('test.key', 'message', {expiration: "2000", persistent: false})
+        broker.publish('test.key', {key: "value"}, {expiration: "2000", persistent: false})
       end
 
       context 'when there are global properties' do
@@ -309,8 +392,8 @@ describe Hutch::Broker do
 
           it 'merges the properties' do
             expect(broker.exchange).
-              to receive(:publish).with('"message"', hash_including(app_id: 'app'))
-            broker.publish('test.key', 'message')
+              to receive(:publish).with('{"key":"value"}', hash_including(app_id: 'app'))
+            broker.publish('test.key', {key: "value"})
           end
         end
 
@@ -321,8 +404,8 @@ describe Hutch::Broker do
 
           it 'calls the proc and merges the properties' do
             expect(broker.exchange).
-              to receive(:publish).with('"message"', hash_including(app_id: 'app'))
-            broker.publish('test.key', 'message')
+              to receive(:publish).with('{"key":"value"}', hash_including(app_id: 'app'))
+            broker.publish('test.key', {key: "value"})
           end
         end
       end
@@ -331,13 +414,13 @@ describe Hutch::Broker do
         it 'does not wait for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to_not receive(:wait_for_confirms)
-          broker.publish('test.key', 'message')
+          broker.publish('test.key', {key: "value"})
         end
 
         it 'does not wait for confirms on the channel', adapter: :march_hare do
           expect_any_instance_of(MarchHare::Channel).
             to_not receive(:wait_for_confirms)
-          broker.publish('test.key', 'message')
+          broker.publish('test.key', {key: "value"})
         end
       end
 
@@ -351,26 +434,28 @@ describe Hutch::Broker do
         it 'waits for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to receive(:wait_for_confirms)
-          broker.publish('test.key', 'message')
+          broker.publish('test.key', {key: "value"})
         end
 
         it 'waits for confirms on the channel', adapter: :march_hare do
           expect_any_instance_of(MarchHare::Channel).
             to receive(:wait_for_confirms)
-          broker.publish('test.key', 'message')
+          broker.publish('test.key', {key: "value"})
         end
       end
     end
 
     context 'without a valid connection' do
+      before { broker.set_up_amqp_connection; broker.disconnect }
+
       it 'raises an exception' do
-        expect { broker.publish('test.key', 'message') }.
+        expect { broker.publish('test.key', {key: "value"}) }.
           to raise_exception(Hutch::PublishError)
       end
 
       it 'logs an error' do
         expect(broker.logger).to receive(:error)
-        broker.publish('test.key', 'message') rescue nil
+        broker.publish('test.key', {key: "value"}) rescue nil
       end
     end
   end