hutch 0.18.0 → 1.1.0

data/lib/hutch/error_handlers/sentry.rb

@@ -1,22 +1,25 @@
 require 'hutch/logging'
-require 'raven'
+require 'sentry-ruby'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Sentry
-      include Logging
-
-      def initialize
-        unless Raven.respond_to?(:capture_exception)
-          raise "The Hutch Sentry error handler requires Raven >= 0.4.0"
+    class Sentry < Base
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} Logging event to Sentry"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+        ::Sentry.configure_scope do |scope|
+          scope.set_context("payload", payload)
         end
+        ::Sentry.capture_exception(ex)
       end
 
-      def handle(message_id, payload, consumer, ex)
-        prefix = "message(#{message_id || '-'}): "
-        logger.error prefix + "Logging event to Sentry"
-        logger.error prefix + "#{ex.class} - #{ex.message}"
-        Raven.capture_exception(ex)
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Sentry"
+        logger.error "#{ex.class} - #{ex.message}"
+        ::Sentry.capture_exception(ex)
       end
     end
   end

data/lib/hutch/error_handlers/sentry_raven.rb

@@ -0,0 +1,31 @@
+require 'hutch/logging'
+require 'raven'
+require 'hutch/error_handlers/base'
+
+module Hutch
+  module ErrorHandlers
+    class SentryRaven < Base
+
+      def initialize
+        unless Raven.respond_to?(:capture_exception)
+          raise "The Hutch Sentry error handler requires Raven >= 0.4.0"
+        end
+      end
+
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} Logging event to Sentry"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+        Raven.capture_exception(ex, extra: { payload: payload })
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Sentry"
+        logger.error "#{ex.class} - #{ex.message}"
+        Raven.capture_exception(ex)
+      end
+
+    end
+  end
+end

data/lib/hutch/error_handlers.rb

@@ -2,7 +2,10 @@ module Hutch
   module ErrorHandlers
     autoload :Logger,      'hutch/error_handlers/logger'
     autoload :Sentry,      'hutch/error_handlers/sentry'
+    autoload :SentryRaven, 'hutch/error_handlers/sentry_raven'
     autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
     autoload :Airbrake,    'hutch/error_handlers/airbrake'
+    autoload :Rollbar,     'hutch/error_handlers/rollbar'
+    autoload :Bugsnag,     'hutch/error_handlers/bugsnag'
   end
 end

data/lib/hutch/exceptions.rb

@@ -1,5 +1,12 @@
 module Hutch
-  class Exception < StandardError; end
+  if defined?(JRUBY_VERSION)
+    require 'march_hare/exceptions'
+    class Exception < MarchHare::Exception; end
+  else
+    require "bunny/exceptions"
+    # Bunny::Exception inherits from StandardError
+    class Exception < Bunny::Exception; end
+  end
   class ConnectionError < Exception; end
   class AuthenticationError < Exception; end
   class WorkerSetupError < Exception; end

data/lib/hutch/logging.rb

@@ -9,12 +9,12 @@ module Hutch
       end
     end
 
-    def self.setup_logger(target = $stdout)
+    def self.setup_logger
       require 'hutch/config'
-      @logger = Logger.new(target)
-      @logger.level = Hutch::Config.log_level
-      @logger.formatter = HutchFormatter.new
-      @logger
+      @logger = Logger.new($stdout).tap do |l|
+        l.level = Hutch::Config.log_level
+        l.formatter = HutchFormatter.new
+      end
     end
 
     def self.logger

data/lib/hutch/message.rb

@@ -1,6 +1,4 @@
-require 'multi_json'
 require 'forwardable'
-require 'active_support/core_ext/hash/indifferent_access'
 
 module Hutch
   class Message
@@ -8,11 +6,11 @@ module Hutch
 
     attr_reader :delivery_info, :properties, :payload
 
-    def initialize(delivery_info, properties, payload)
+    def initialize(delivery_info, properties, payload, serializer)
       @delivery_info = delivery_info
       @properties    = properties
       @payload       = payload
-      @body          = MultiJson.load(payload).with_indifferent_access
+      @body          = serializer.decode(payload)
     end
 
     def_delegator :@body, :[]

data/lib/hutch/publisher.rb

@@ -0,0 +1,75 @@
+require 'securerandom'
+require 'hutch/logging'
+require 'hutch/exceptions'
+
+module Hutch
+  class Publisher
+    include Logging
+    attr_reader :connection, :channel, :exchange, :config
+
+    def initialize(connection, channel, exchange, config = Hutch::Config)
+      @connection = connection
+      @channel    = channel
+      @exchange   = exchange
+      @config     = config
+    end
+
+    def publish(routing_key, message, properties = {}, options = {})
+      ensure_connection!(routing_key, message)
+
+      serializer = options[:serializer] || config[:serializer]
+
+      non_overridable_properties = {
+        routing_key:  routing_key,
+        timestamp:    connection.current_timestamp,
+        content_type: serializer.content_type,
+      }
+      properties[:message_id]   ||= generate_id
+
+      payload = serializer.encode(message)
+
+      log_publication(serializer, payload, routing_key)
+
+      response = exchange.publish(payload, {persistent: true}.
+        merge(properties).
+        merge(global_properties).
+        merge(non_overridable_properties))
+
+      channel.wait_for_confirms if config[:force_publisher_confirms]
+      response
+    end
+
+    private
+
+    def log_publication(serializer, payload, routing_key)
+      logger.debug {
+        spec =
+          if serializer.binary?
+            "#{payload.bytesize} bytes message"
+          else
+            "message '#{payload}'"
+          end
+        "publishing #{spec} to #{routing_key}"
+      }
+    end
+
+    def raise_publish_error(reason, routing_key, message)
+      msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
+      logger.error(msg)
+      raise PublishError, msg
+    end
+
+    def ensure_connection!(routing_key, message)
+      raise_publish_error('no connection to broker', routing_key, message) unless connection
+      raise_publish_error('connection is closed', routing_key, message) unless connection.open?
+    end
+
+    def generate_id
+      SecureRandom.uuid
+    end
+
+    def global_properties
+      Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
+    end
+  end
+end

data/lib/hutch/serializers/identity.rb

@@ -0,0 +1,19 @@
+module Hutch
+  module Serializers
+    class Identity
+
+      def self.encode(payload)
+        payload
+      end
+
+      def self.decode(payload)
+        payload
+      end
+
+      def self.binary? ; false ; end
+
+      def self.content_type ; nil ; end
+
+    end
+  end
+end

data/lib/hutch/serializers/json.rb

@@ -0,0 +1,22 @@
+require 'multi_json'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Hutch
+  module Serializers
+    class JSON
+
+      def self.encode(payload)
+        ::MultiJson.dump(payload)
+      end
+
+      def self.decode(payload)
+        ::MultiJson.load(payload).with_indifferent_access
+      end
+
+      def self.binary? ; false ; end
+
+      def self.content_type ; 'application/json' ; end
+
+    end
+  end
+end

data/lib/hutch/tracers/datadog.rb

@@ -0,0 +1,17 @@
+require 'ddtrace'
+
+module Hutch
+  module Tracers
+    class Datadog
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        ::Datadog.tracer.trace(@klass.class.name, service: 'hutch', span_type: 'rabbitmq') do
+          @klass.process(message)
+        end
+      end
+    end
+  end
+end

data/lib/hutch/tracers.rb

@@ -2,5 +2,6 @@ module Hutch
   module Tracers
     autoload :NullTracer, 'hutch/tracers/null_tracer'
     autoload :NewRelic,   'hutch/tracers/newrelic'
+    autoload :Datadog,    'hutch/tracers/datadog'
   end
 end

data/lib/hutch/version.rb

@@ -1,4 +1,3 @@
 module Hutch
-  VERSION = '0.18.0'.freeze
+  VERSION = '1.1.0'.freeze
 end
-

data/lib/hutch/waiter.rb

@@ -0,0 +1,104 @@
+require 'hutch/logging'
+
+module Hutch
+  # Signal-handling class.
+  #
+  # Currently, the signal USR2 performs a thread dump,
+  # while QUIT, TERM and INT all perform a graceful shutdown.
+  class Waiter
+    include Logging
+
+    class ContinueProcessingSignals < RuntimeError
+    end
+
+    def self.supported_signals_of(list)
+      list.keep_if { |s| Signal.list.keys.include?(s) }.tap do |result|
+        result.delete('QUIT') if defined?(JRUBY_VERSION)
+      end
+    end
+
+    SHUTDOWN_SIGNALS = supported_signals_of(%w(QUIT TERM INT)).freeze
+    # We have chosen a JRuby-supported signal
+    USER_SIGNALS = supported_signals_of(%w(USR2)).freeze
+    REGISTERED_SIGNALS = (SHUTDOWN_SIGNALS + USER_SIGNALS).freeze
+
+    def self.wait_until_signaled
+      new.wait_until_signaled
+    end
+
+    def wait_until_signaled
+      self.sig_read, self.sig_write = IO.pipe
+
+      register_signal_handlers
+
+      begin
+        wait_for_signal
+
+        sig = sig_read.gets.strip
+        handle_signal(sig)
+      rescue ContinueProcessingSignals
+        retry
+      end
+    end
+
+    def handle_signal(sig)
+      raise ContinueProcessingSignals unless REGISTERED_SIGNALS.include?(sig)
+      if user_signal?(sig)
+        handle_user_signal(sig)
+      else
+        handle_shutdown_signal(sig)
+      end
+    end
+
+    # @raise ContinueProcessingSignals
+    def handle_user_signal(sig)
+      case sig
+      when 'USR2' then log_thread_backtraces
+      else raise "Assertion failed - unhandled signal: #{sig.inspect}"
+      end
+      raise ContinueProcessingSignals
+    end
+
+    def handle_shutdown_signal(sig)
+      logger.info "caught SIG#{sig}, stopping hutch..."
+    end
+
+    private
+
+    def log_thread_backtraces
+      logger.info 'Requested a VM-wide thread stack trace dump...'
+      Thread.list.each do |thread|
+        logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+        logger.info backtrace_for(thread)
+      end
+    end
+
+    def backtrace_for(thread)
+      if thread.backtrace
+        thread.backtrace.join("\n")
+      else
+        '<no backtrace available>'
+      end
+    end
+
+    attr_accessor :sig_read, :sig_write
+
+    def wait_for_signal
+      IO.select([sig_read])
+    end
+
+    def register_signal_handlers
+      REGISTERED_SIGNALS.each do |sig|
+        # This needs to be reentrant, so we queue up signals to be handled
+        # in the run loop, rather than acting on signals here
+        trap(sig) do
+          sig_write.puts(sig)
+        end
+      end
+    end
+
+    def user_signal?(sig)
+      USER_SIGNALS.include?(sig)
+    end
+  end
+end

data/lib/hutch/worker.rb

@@ -1,15 +1,19 @@
 require 'hutch/message'
 require 'hutch/logging'
 require 'hutch/broker'
+require 'hutch/acknowledgements/nack_on_all_failures'
+require 'hutch/waiter'
 require 'carrot-top'
+require 'securerandom'
 
 module Hutch
   class Worker
     include Logging
 
-    def initialize(broker, consumers)
+    def initialize(broker, consumers, setup_procs)
       @broker        = broker
       self.consumers = consumers
+      self.setup_procs = setup_procs
     end
 
     # Run the main event loop. The consumers will be set up with queues, and
@@ -17,46 +21,11 @@ module Hutch
     # never returns.
     def run
       setup_queues
+      setup_procs.each(&:call)
 
-      # Set up signal handlers for graceful shutdown
-      register_signal_handlers
+      Waiter.wait_until_signaled
 
-      main_loop
-    end
-
-    def main_loop
-      if defined?(JRUBY_VERSION)
-        # Binds shutdown listener to notify main thread if channel was closed
-        bind_shutdown_handler
-
-        handle_signals until shutdown_not_called?(0.1)
-      else
-        # Take a break from Thread#join every 0.1 seconds to check if we've
-        # been sent any signals
-        handle_signals until @broker.wait_on_threads(0.1)
-      end
-    end
-
-    # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
-    # gracefully. Forceful shutdowns are very bad!
-    def register_signal_handlers
-      Thread.main[:signal_queue] = []
-      %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.map(&:to_sym).each do |sig|
-        # This needs to be reentrant, so we queue up signals to be handled
-        # in the run loop, rather than acting on signals here
-        trap(sig) do
-          Thread.main[:signal_queue] << sig
-        end
-      end
-    end
-
-    # Handle any pending signals
-    def handle_signals
-      signal = Thread.main[:signal_queue].shift
-      if signal
-        logger.info "caught sig#{signal.downcase}, stopping hutch..."
-        stop
-      end
+      stop
     end
 
     # Stop a running worker by killing all subscriber threads.
@@ -64,36 +33,24 @@ module Hutch
       @broker.stop
     end
 
-    # Binds shutdown handler, called if channel is closed or network Failed
-    def bind_shutdown_handler
-      @broker.channel.on_shutdown do
-        Thread.main[:shutdown_received] = true
-      end
-    end
-
-    # Checks if shutdown handler was called, then sleeps for interval
-    def shutdown_not_called?(interval)
-      if Thread.main[:shutdown_received]
-        true
-      else
-        sleep(interval)
-        false
-      end
-    end
-
     # Set up the queues for each of the worker's consumers.
     def setup_queues
       logger.info 'setting up queues'
-      @consumers.each { |consumer| setup_queue(consumer) }
+      vetted = @consumers.reject { |c| group_configured? && group_restricted?(c) }
+      vetted.each do |c|
+        setup_queue(c)
+      end
     end
 
     # Bind a consumer's routing keys to its queue, and set up a subscription to
     # receive messages sent to the queue.
     def setup_queue(consumer)
+      logger.info "setting up queue: #{consumer.get_queue_name}"
+
       queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
       @broker.bind_queue(queue, consumer.routing_keys)
 
-      queue.subscribe(manual_ack: true) do |*args|
+      queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
         delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
         handle_message(consumer, delivery_info, properties, payload)
       end
@@ -102,30 +59,39 @@ module Hutch
     # Called internally when a new messages comes in from RabbitMQ. Responsible
     # for wrapping up the message and passing it to the consumer.
     def handle_message(consumer, delivery_info, properties, payload)
-      logger.info("message(#{properties.message_id || '-'}): " +
-                  "routing key: #{delivery_info.routing_key}, " +
-                  "consumer: #{consumer}, " +
-                  "payload: #{payload}")
-
-      broker = @broker
-      begin
-        message = Message.new(delivery_info, properties, payload)
-        consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
-        with_tracing(consumer_instance).handle(message)
-        broker.ack(delivery_info.delivery_tag)
-      rescue StandardError => ex
-        broker.nack(delivery_info.delivery_tag)
-        handle_error(properties.message_id, payload, consumer, ex)
-      end
+      serializer = consumer.get_serializer || Hutch::Config[:serializer]
+      logger.debug {
+        spec   = serializer.binary? ? "#{payload.bytesize} bytes" : "#{payload}"
+        "message(#{properties.message_id || '-'}): " +
+        "routing key: #{delivery_info.routing_key}, " +
+        "consumer: #{consumer}, " +
+        "payload: #{spec}"
+      }
+
+      message = Message.new(delivery_info, properties, payload, serializer)
+      consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
+      with_tracing(consumer_instance).handle(message)
+      @broker.ack(delivery_info.delivery_tag)
+    rescue => ex
+      acknowledge_error(delivery_info, properties, @broker, ex)
+      handle_error(properties, payload, consumer, ex)
     end
 
     def with_tracing(klass)
       Hutch::Config[:tracer].new(klass)
     end
 
-    def handle_error(message_id, payload, consumer, ex)
+    def handle_error(*args)
       Hutch::Config[:error_handlers].each do |backend|
-        backend.handle(message_id, payload, consumer, ex)
+        backend.handle(*args)
+      end
+    end
+
+    def acknowledge_error(delivery_info, properties, broker, ex)
+      acks = error_acknowledgements +
+        [Hutch::Acknowledgements::NackOnAllFailures.new]
+      acks.find do |backend|
+        backend.handle(delivery_info, properties, broker, ex)
       end
     end
 
@@ -135,5 +101,45 @@ module Hutch
       end
       @consumers = val
     end
+
+    def error_acknowledgements
+      Hutch::Config[:error_acknowledgements]
+    end
+
+    private
+
+    def group_configured?
+      if group.present? && consumer_groups.blank?
+        logger.info 'Consumer groups are blank'
+      end
+      group.present?
+    end
+
+    def group_restricted?(consumer)
+      consumers_to_load = consumer_groups[group]
+      if consumers_to_load
+        !consumers_to_load.include?(consumer.name)
+      else
+        true
+      end
+    end
+
+    def group
+      Hutch::Config[:group]
+    end
+
+    def consumer_groups
+      Hutch::Config[:consumer_groups]
+    end
+
+    attr_accessor :setup_procs
+
+    def unique_consumer_tag
+      prefix = Hutch::Config[:consumer_tag_prefix]
+      unique_part = SecureRandom.uuid
+      "#{prefix}-#{unique_part}".tap do |tag|
+        raise "Tag must be 255 bytes long at most, current one is #{tag.bytesize} ('#{tag}')" if tag.bytesize > 255
+      end
+    end
   end
 end

data/lib/hutch.rb

@@ -3,6 +3,8 @@ require 'hutch/consumer'
 require 'hutch/worker'
 require 'hutch/broker'
 require 'hutch/logging'
+require 'hutch/serializers/identity'
+require 'hutch/serializers/json'
 require 'hutch/config'
 require 'hutch/message'
 require 'hutch/cli'
@@ -12,6 +14,7 @@ require 'hutch/exceptions'
 require 'hutch/tracers'
 
 module Hutch
+  @@connection_mutex = Mutex.new
 
   def self.register_consumer(consumer)
     self.consumers << consumer
@@ -33,10 +36,17 @@ module Hutch
     @global_properties ||= {}
   end
 
+  # Connects to broker, if not yet connected.
+  #
+  # @param options [Hash] Connection options
+  # @param config [Hash] Configuration
+  # @option options [Boolean] :enable_http_api_use
   def self.connect(options = {}, config = Hutch::Config)
-    unless connected?
-      @broker = Hutch::Broker.new(config)
-      @broker.connect(options)
+    @@connection_mutex.synchronize do
+      unless connected?
+        @broker = Hutch::Broker.new(config)
+        @broker.connect(options)
+      end
     end
   end
 
@@ -48,10 +58,9 @@ module Hutch
     @broker
   end
 
+  # @return [Boolean]
   def self.connected?
-    return false unless broker
-    return false unless broker.connection
-    broker.connection.open?
+    broker && broker.connection && broker.connection.open?
   end
 
   def self.publish(*args)