hutch 0.19.0-java

data/lib/hutch/error_handlers/logger.rb

@@ -0,0 +1,16 @@
+require 'hutch/logging'
+
+module Hutch
+  module ErrorHandlers
+    class Logger
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "error in consumer '#{consumer}'"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        logger.error (['backtrace:'] + ex.backtrace).join("\n")
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/sentry.rb

@@ -0,0 +1,23 @@
+require 'hutch/logging'
+require 'raven'
+
+module Hutch
+  module ErrorHandlers
+    class Sentry
+      include Logging
+
+      def initialize
+        unless Raven.respond_to?(:capture_exception)
+          raise "The Hutch Sentry error handler requires Raven >= 0.4.0"
+        end
+      end
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Sentry"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        Raven.capture_exception(ex)
+      end
+    end
+  end
+end

data/lib/hutch/exceptions.rb

@@ -0,0 +1,7 @@
+module Hutch
+  class Exception < StandardError; end
+  class ConnectionError < Exception; end
+  class AuthenticationError < Exception; end
+  class WorkerSetupError < Exception; end
+  class PublishError < Exception; end
+end

data/lib/hutch/logging.rb

@@ -0,0 +1,32 @@
+require 'logger'
+require 'time'
+
+module Hutch
+  module Logging
+    class HutchFormatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} #{severity} -- #{message}\n"
+      end
+    end
+
+    def self.setup_logger(target = $stdout)
+      require 'hutch/config'
+      @logger = Logger.new(target)
+      @logger.level = Hutch::Config.log_level
+      @logger.formatter = HutchFormatter.new
+      @logger
+    end
+
+    def self.logger
+      @logger || setup_logger
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def logger
+      Hutch::Logging.logger
+    end
+  end
+end

data/lib/hutch/message.rb

@@ -0,0 +1,31 @@
+require 'forwardable'
+
+module Hutch
+  class Message
+    extend Forwardable
+
+    attr_reader :delivery_info, :properties, :payload
+
+    def initialize(delivery_info, properties, payload, serializer)
+      @delivery_info = delivery_info
+      @properties    = properties
+      @payload       = payload
+      @body          = serializer.decode(payload)
+    end
+
+    def_delegator :@body, :[]
+    def_delegators :@properties, :message_id, :timestamp
+    def_delegators :@delivery_info, :routing_key, :exchange
+
+    attr_reader :body
+
+    def to_s
+      attrs = { :@body => body.to_s, message_id: message_id,
+                timestamp: timestamp, routing_key: routing_key }
+      "#<Message #{attrs.map { |k,v| "#{k}=#{v.inspect}" }.join(', ')}>"
+    end
+
+    alias_method :inspect, :to_s
+  end
+end
+

data/lib/hutch/serializers/identity.rb

@@ -0,0 +1,19 @@
+module Hutch
+  module Serializers
+    class Identity
+
+      def self.encode(payload)
+        payload
+      end
+
+      def self.decode(payload)
+        payload
+      end
+
+      def self.binary? ; false ; end
+
+      def self.content_type ; nil ; end
+
+    end
+  end
+end

data/lib/hutch/serializers/json.rb

@@ -0,0 +1,22 @@
+require 'multi_json'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Hutch
+  module Serializers
+    class JSON
+
+      def self.encode(payload)
+        ::JSON.dump(payload)
+      end
+
+      def self.decode(payload)
+        ::MultiJson.load(payload).with_indifferent_access
+      end
+
+      def self.binary? ; false ; end
+
+      def self.content_type ; 'application/json' ; end
+
+    end
+  end
+end

data/lib/hutch/tracers.rb

@@ -0,0 +1,6 @@
+module Hutch
+  module Tracers
+    autoload :NullTracer, 'hutch/tracers/null_tracer'
+    autoload :NewRelic,   'hutch/tracers/newrelic'
+  end
+end

data/lib/hutch/tracers/newrelic.rb

@@ -0,0 +1,19 @@
+require 'newrelic_rpm'
+
+module Hutch
+  module Tracers
+    class NewRelic
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        @klass.process(message)
+      end
+
+      add_transaction_tracer :handle, :category => 'OtherTransaction/HutchConsumer', :path => '#{@klass.class.name}'
+    end
+  end
+end

data/lib/hutch/tracers/null_tracer.rb

@@ -0,0 +1,15 @@
+module Hutch
+  module Tracers
+    class NullTracer
+
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        @klass.process(message)
+      end
+
+    end
+  end
+end

data/lib/hutch/version.rb

@@ -0,0 +1,4 @@
+module Hutch
+  VERSION = '0.19.0'.freeze
+end
+

data/lib/hutch/worker.rb

@@ -0,0 +1,143 @@
+require 'hutch/message'
+require 'hutch/logging'
+require 'hutch/broker'
+require 'carrot-top'
+
+module Hutch
+  class Worker
+    include Logging
+
+    def initialize(broker, consumers)
+      @broker        = broker
+      self.consumers = consumers
+    end
+
+    # Run the main event loop. The consumers will be set up with queues, and
+    # process the messages in their respective queues indefinitely. This method
+    # never returns.
+    def run
+      setup_queues
+
+      # Set up signal handlers for graceful shutdown
+      register_signal_handlers
+
+      main_loop
+    end
+
+    def main_loop
+      if defined?(JRUBY_VERSION)
+        # Binds shutdown listener to notify main thread if channel was closed
+        bind_shutdown_handler
+
+        handle_signals until shutdown_not_called?(0.1)
+      else
+        # Take a break from Thread#join every 0.1 seconds to check if we've
+        # been sent any signals
+        handle_signals until @broker.wait_on_threads(0.1)
+      end
+    end
+
+    # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
+    # gracefully. Forceful shutdowns are very bad!
+    def register_signal_handlers
+      Thread.main[:signal_queue] = []
+      %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.map(&:to_sym).each do |sig|
+        # This needs to be reentrant, so we queue up signals to be handled
+        # in the run loop, rather than acting on signals here
+        trap(sig) do
+          Thread.main[:signal_queue] << sig
+        end
+      end
+    end
+
+    # Handle any pending signals
+    def handle_signals
+      signal = Thread.main[:signal_queue].shift
+      if signal
+        logger.info "caught sig#{signal.downcase}, stopping hutch..."
+        stop
+      end
+    end
+
+    # Stop a running worker by killing all subscriber threads.
+    def stop
+      @broker.stop
+    end
+
+    # Binds shutdown handler, called if channel is closed or network Failed
+    def bind_shutdown_handler
+      @broker.channel.on_shutdown do
+        Thread.main[:shutdown_received] = true
+      end
+    end
+
+    # Checks if shutdown handler was called, then sleeps for interval
+    def shutdown_not_called?(interval)
+      if Thread.main[:shutdown_received]
+        true
+      else
+        sleep(interval)
+        false
+      end
+    end
+
+    # Set up the queues for each of the worker's consumers.
+    def setup_queues
+      logger.info 'setting up queues'
+      @consumers.each { |consumer| setup_queue(consumer) }
+    end
+
+    # Bind a consumer's routing keys to its queue, and set up a subscription to
+    # receive messages sent to the queue.
+    def setup_queue(consumer)
+      queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
+      @broker.bind_queue(queue, consumer.routing_keys)
+
+      queue.subscribe(manual_ack: true) do |*args|
+        delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
+        handle_message(consumer, delivery_info, properties, payload)
+      end
+    end
+
+    # Called internally when a new messages comes in from RabbitMQ. Responsible
+    # for wrapping up the message and passing it to the consumer.
+    def handle_message(consumer, delivery_info, properties, payload)
+      broker = @broker
+      begin
+        serializer = consumer.get_serializer || Hutch::Config[:serializer]
+        logger.info {
+          spec   = serializer.binary? ? "#{payload.bytesize} bytes" : "#{payload}"
+          "message(#{properties.message_id || '-'}): " +
+          "routing key: #{delivery_info.routing_key}, " +
+          "consumer: #{consumer}, " +
+          "payload: #{spec}"
+        }
+
+        message = Message.new(delivery_info, properties, payload, serializer)
+        consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
+        with_tracing(consumer_instance).handle(message)
+        broker.ack(delivery_info.delivery_tag)
+      rescue StandardError => ex
+        broker.nack(delivery_info.delivery_tag)
+        handle_error(properties.message_id, payload, consumer, ex)
+      end
+    end
+
+    def with_tracing(klass)
+      Hutch::Config[:tracer].new(klass)
+    end
+
+    def handle_error(message_id, payload, consumer, ex)
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle(message_id, payload, consumer, ex)
+      end
+    end
+
+    def consumers=(val)
+      if val.empty?
+        logger.warn "no consumer loaded, ensure there's no configuration issue"
+      end
+      @consumers = val
+    end
+  end
+end

data/spec/hutch/broker_spec.rb

@@ -0,0 +1,377 @@
+require 'spec_helper'
+require 'hutch/broker'
+
+describe Hutch::Broker do
+  let(:config) { deep_copy(Hutch::Config.user_config) }
+  subject(:broker) { Hutch::Broker.new(config) }
+
+  describe '#connect' do
+    before { allow(broker).to receive(:set_up_amqp_connection) }
+    before { allow(broker).to receive(:set_up_api_connection) }
+    before { allow(broker).to receive(:disconnect) }
+
+    it 'sets up the amqp connection' do
+      expect(broker).to receive(:set_up_amqp_connection)
+      broker.connect
+    end
+
+    it 'sets up the api connection' do
+      expect(broker).to receive(:set_up_api_connection)
+      broker.connect
+    end
+
+    it 'does not disconnect' do
+      expect(broker).not_to receive(:disconnect)
+      broker.connect
+    end
+
+    context 'when given a block' do
+      it 'disconnects' do
+        expect(broker).to receive(:disconnect).once
+        broker.connect { }
+      end
+    end
+
+    context 'when given a block that fails' do
+      let(:exception) { Class.new(StandardError) }
+
+      it 'disconnects' do
+        expect(broker).to receive(:disconnect).once
+        expect do
+          broker.connect { fail exception }
+        end.to raise_error(exception)
+      end
+    end
+
+    context "with options" do
+      let(:options) { { enable_http_api_use: false } }
+
+      it "doesnt set up api" do
+        expect(broker).not_to receive(:set_up_api_connection)
+        broker.connect options
+      end
+    end
+  end
+
+  describe '#set_up_amqp_connection', rabbitmq: true do
+    context 'with valid details' do
+      before { broker.set_up_amqp_connection }
+      after  { broker.disconnect }
+
+      describe '#connection', adapter: :bunny do
+        subject { super().connection }
+        it { is_expected.to be_a Hutch::Adapters::BunnyAdapter }
+      end
+
+      describe '#connection', adapter: :march_hare do
+        subject { super().connection }
+        it { is_expected.to be_a Hutch::Adapters::MarchHareAdapter }
+      end
+
+      describe '#channel', adapter: :bunny do
+        subject { super().channel }
+        it { is_expected.to be_a Bunny::Channel }
+      end
+
+      describe '#channel', adapter: :march_hare do
+        subject { super().channel }
+        it { is_expected.to be_a MarchHare::Channel }
+      end
+
+      describe '#exchange', adapter: :bunny do
+        subject { super().exchange }
+        it { is_expected.to be_a Bunny::Exchange }
+      end
+
+      describe '#exchange', adapter: :march_hare do
+        subject { super().exchange }
+        it { is_expected.to be_a MarchHare::Exchange }
+      end
+    end
+
+    context 'when given invalid details' do
+      before { config[:mq_host] = 'notarealhost' }
+      let(:set_up_amqp_connection) { ->{ broker.set_up_amqp_connection } }
+
+      specify { expect(set_up_amqp_connection).to raise_error }
+    end
+
+    context 'with channel_prefetch set' do
+      let(:prefetch_value) { 1 }
+      before { config[:channel_prefetch] = prefetch_value }
+      after  { broker.disconnect }
+
+      it "set's channel's prefetch", adapter: :bunny do
+        expect_any_instance_of(Bunny::Channel).
+          to receive(:prefetch).with(prefetch_value)
+        broker.set_up_amqp_connection
+      end
+
+      it "set's channel's prefetch", adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:prefetch=).with(prefetch_value)
+        broker.set_up_amqp_connection
+      end
+    end
+
+    context 'with force_publisher_confirms set' do
+      let(:force_publisher_confirms_value) { true }
+      before { config[:force_publisher_confirms] = force_publisher_confirms_value }
+      after  { broker.disconnect }
+
+      it 'waits for confirmation', adapter: :bunny do
+        expect_any_instance_of(Bunny::Channel).
+          to receive(:confirm_select)
+        broker.set_up_amqp_connection
+      end
+
+      it 'waits for confirmation', adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:confirm_select)
+        broker.set_up_amqp_connection
+      end
+    end
+  end
+
+  describe '#set_up_api_connection', rabbitmq: true do
+    context 'with valid details' do
+      before { broker.set_up_api_connection }
+      after  { broker.disconnect }
+
+      describe '#api_client' do
+        subject { super().api_client }
+        it { is_expected.to be_a CarrotTop }
+      end
+    end
+
+    context 'when given invalid details' do
+      before { config[:mq_api_host] = 'notarealhost' }
+      after  { broker.disconnect }
+      let(:set_up_api_connection) { ->{ broker.set_up_api_connection } }
+
+      specify { expect(set_up_api_connection).to raise_error }
+    end
+  end
+
+  describe '#queue' do
+    let(:channel) { double('Channel') }
+    let(:arguments) { { foo: :bar } }
+    before { allow(broker).to receive(:channel) { channel } }
+
+    it 'applies a global namespace' do
+      config[:namespace] = 'mirror-all.service'
+      expect(broker.channel).to receive(:queue) do |*args|
+        args.first == ''
+        args.last == arguments
+      end
+      broker.queue('test', arguments)
+    end
+  end
+
+  describe '#bindings', rabbitmq: true do
+    around { |example| broker.connect { example.run } }
+    subject { broker.bindings }
+
+    context 'with no bindings' do
+      describe '#keys' do
+        subject { super().keys }
+        it { is_expected.not_to include 'test' }
+      end
+    end
+
+    context 'with a binding' do
+      around do |example|
+        queue = broker.queue('test').bind(broker.exchange, routing_key: 'key')
+        example.run
+        queue.unbind(broker.exchange, routing_key: 'key').delete
+      end
+
+      it { is_expected.to include({ 'test' => ['key'] }) }
+    end
+  end
+
+  describe '#bind_queue' do
+
+    around { |example| broker.connect { example.run } }
+
+    let(:routing_keys) { %w( a b c ) }
+    let(:queue) { double('Queue', bind: nil, unbind: nil, name: 'consumer') }
+    before { allow(broker).to receive(:bindings).and_return('consumer' => ['d']) }
+
+    it 'calls bind for each routing key' do
+      routing_keys.each do |key|
+        expect(queue).to receive(:bind).with(broker.exchange, routing_key: key)
+      end
+      broker.bind_queue(queue, routing_keys)
+    end
+
+    it 'calls unbind for each redundant existing binding' do
+      expect(queue).to receive(:unbind).with(broker.exchange, routing_key: 'd')
+      broker.bind_queue(queue, routing_keys)
+    end
+
+    context '(rabbitmq integration test)', rabbitmq: true do
+      let(:queue) { broker.queue('consumer') }
+      let(:routing_key) { 'key' }
+
+      before { allow(broker).to receive(:bindings).and_call_original }
+      before { queue.bind(broker.exchange, routing_key: 'redundant-key') }
+      after { queue.unbind(broker.exchange, routing_key: routing_key).delete }
+
+      it 'results in the correct bindings' do
+        broker.bind_queue(queue, [routing_key])
+        expect(broker.bindings).to include({ queue.name => [routing_key] })
+      end
+    end
+  end
+
+  describe '#wait_on_threads' do
+    let(:thread) { double('Thread') }
+    before { allow(broker).to receive(:work_pool_threads).and_return(threads) }
+
+    context 'when all threads finish within the timeout' do
+      let(:threads) { [double(join: thread), double(join: thread)] }
+      specify { expect(broker.wait_on_threads(1)).to be_truthy }
+    end
+
+    context 'when timeout expires for one thread' do
+      let(:threads) { [double(join: thread), double(join: nil)] }
+      specify { expect(broker.wait_on_threads(1)).to be_falsey }
+    end
+  end
+
+  describe '#stop', adapter: :bunny do
+    let(:thread_1) { double('Thread') }
+    let(:thread_2) { double('Thread') }
+    let(:work_pool) { double('Bunny::ConsumerWorkPool') }
+    let(:config) { { graceful_exit_timeout: 2 } }
+
+    before do
+      allow(broker).to receive(:channel_work_pool).and_return(work_pool)
+    end
+
+    it 'gracefully stops the work pool' do
+      expect(work_pool).to receive(:shutdown)
+      expect(work_pool).to receive(:join).with(2)
+      expect(work_pool).to receive(:kill)
+
+      broker.stop
+    end
+  end
+
+  describe '#stop', adapter: :march_hare do
+    let(:channel) { double('MarchHare::Channel')}
+
+    before do
+      allow(broker).to receive(:channel).and_return(channel)
+    end
+
+    it 'gracefully stops the channel' do
+      expect(channel).to receive(:close)
+
+      broker.stop
+    end
+  end
+
+  describe '#publish' do
+    context 'with a valid connection' do
+      before { broker.set_up_amqp_connection }
+      after  { broker.disconnect }
+
+      it 'publishes to the exchange' do
+        expect(broker.exchange).to receive(:publish).once
+        broker.publish('test.key', 'message')
+      end
+
+      it 'sets default properties' do
+        expect(broker.exchange).to receive(:publish).with(
+          JSON.dump("message"),
+          hash_including(
+            persistent: true,
+            routing_key: 'test.key',
+            content_type: 'application/json'
+          )
+        )
+
+        broker.publish('test.key', 'message')
+      end
+
+      it 'allows passing message properties' do
+        expect(broker.exchange).to receive(:publish).once
+        broker.publish('test.key', 'message', {expiration: "2000", persistent: false})
+      end
+
+      context 'when there are global properties' do
+        context 'as a hash' do
+          before do
+            allow(Hutch).to receive(:global_properties).and_return(app_id: 'app')
+          end
+
+          it 'merges the properties' do
+            expect(broker.exchange).
+              to receive(:publish).with('"message"', hash_including(app_id: 'app'))
+            broker.publish('test.key', 'message')
+          end
+        end
+
+        context 'as a callable object' do
+          before do
+            allow(Hutch).to receive(:global_properties).and_return(proc { { app_id: 'app' } })
+          end
+
+          it 'calls the proc and merges the properties' do
+            expect(broker.exchange).
+              to receive(:publish).with('"message"', hash_including(app_id: 'app'))
+            broker.publish('test.key', 'message')
+          end
+        end
+      end
+
+      context 'with force_publisher_confirms not set in the config' do
+        it 'does not wait for confirms on the channel', adapter: :bunny do
+          expect_any_instance_of(Bunny::Channel).
+            to_not receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
+
+        it 'does not wait for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to_not receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
+      end
+
+      context 'with force_publisher_confirms set in the config' do
+        let(:force_publisher_confirms_value) { true }
+
+        before do
+          config[:force_publisher_confirms] = force_publisher_confirms_value
+        end
+
+        it 'waits for confirms on the channel', adapter: :bunny do
+          expect_any_instance_of(Bunny::Channel).
+            to receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
+
+        it 'waits for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
+      end
+    end
+
+    context 'without a valid connection' do
+      it 'raises an exception' do
+        expect { broker.publish('test.key', 'message') }.
+          to raise_exception(Hutch::PublishError)
+      end
+
+      it 'logs an error' do
+        expect(broker.logger).to receive(:error)
+        broker.publish('test.key', 'message') rescue nil
+      end
+    end
+  end
+end