hutch 0.18.0 → 1.1.0

data/lib/hutch/config.rb

@@ -1,69 +1,237 @@
 require 'hutch/error_handlers/logger'
+require 'hutch/tracers'
+require 'hutch/serializers/json'
 require 'erb'
 require 'logger'
 
 module Hutch
   class UnknownAttributeError < StandardError; end
 
+  # Configuration settings, available everywhere
+  #
+  # There are defaults, which can be overridden by ENV variables prefixed by
+  # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
+  # method.
+  #
+  # @example Configuring on the command-line
+  #   HUTCH_PUBLISHER_CONFIRMS=false hutch
   module Config
     require 'yaml'
+    @string_keys = Set.new
+    @number_keys = Set.new
+    @boolean_keys = Set.new
+    @settings_defaults = {}
 
-    def self.initialize(params={})
-      @config = {
-        mq_host: 'localhost',
-        mq_port: 5672,
-        mq_exchange: 'hutch',  # TODO: should this be required?
-        mq_vhost: '/',
-        mq_tls: false,
+    # Define a String user setting
+    # @!visibility private
+    def self.string_setting(name, default_value)
+      @string_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Number user setting
+    # @!visibility private
+    def self.number_setting(name, default_value)
+      @number_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Boolean user setting
+    # @!visibility private
+    def self.boolean_setting(name, default_value)
+      @boolean_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # RabbitMQ hostname
+    string_setting :mq_host, '127.0.0.1'
+
+    # RabbitMQ Exchange to use for publishing
+    string_setting :mq_exchange, 'hutch'
+
+    # RabbitMQ Exchange type to use for publishing
+    string_setting :mq_exchange_type, 'topic'
+
+    # RabbitMQ vhost to use
+    string_setting :mq_vhost, '/'
+
+    # RabbitMQ username to use.
+    #
+    # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
+    string_setting :mq_username, 'guest'
+
+    # RabbitMQ password
+    string_setting :mq_password, 'guest'
+
+    # RabbitMQ URI (takes precedence over MQ username, password, host, port and vhost settings)
+    string_setting :uri, nil
+
+    # RabbitMQ HTTP API hostname
+    string_setting :mq_api_host, '127.0.0.1'
+
+    # RabbitMQ port
+    number_setting :mq_port, 5672
+
+    # RabbitMQ HTTP API port
+    number_setting :mq_api_port, 15672
+
+    # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
+    number_setting :heartbeat, 30
+
+    # The <tt>basic.qos</tt> prefetch value to use.
+    #
+    # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
+    number_setting :channel_prefetch, 0
+
+    # Bunny's socket open timeout
+    number_setting :connection_timeout, 11
+
+    # Bunny's socket read timeout
+    number_setting :read_timeout, 11
+
+    # Bunny's socket write timeout
+    number_setting :write_timeout, 11
+
+    # Bunny's enable/disable network recovery
+    boolean_setting :automatically_recover, true
+
+    # Bunny's reconnect interval
+    number_setting :network_recovery_interval, 1
+
+    # FIXME: DOCUMENT THIS
+    number_setting :graceful_exit_timeout, 11
+
+    # Bunny consumer work pool size
+    number_setting :consumer_pool_size, 1
+
+    # Should TLS be used?
+    boolean_setting :mq_tls, false
+
+    # Should SSL certificate be verified?
+    boolean_setting :mq_verify_peer, true
+
+    # Should SSL be used for the RabbitMQ API?
+    boolean_setting :mq_api_ssl, false
+
+    # Should the current Rails app directory be required?
+    boolean_setting :autoload_rails, true
+
+    # Should the Hutch runner process daemonise?
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :daemonise, false
+
+    # Should RabbitMQ publisher confirms be enabled?
+    #
+    # Leaves it up to the app how they are tracked
+    # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
+    boolean_setting :publisher_confirms, false
+
+    # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
+    # every publish.
+    #
+    # **This is the safest option which also offers the
+    # lowest throughput**.
+    boolean_setting :force_publisher_confirms, false
+
+    # Should the RabbitMQ HTTP API be used?
+    boolean_setting :enable_http_api_use, true
+
+    # Should Bunny's consumer work pool threads abort on exception.
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :consumer_pool_abort_on_exception, false
+
+    # Prefix displayed on the consumers tags.
+    string_setting :consumer_tag_prefix, 'hutch'
+
+    # A namespace to help group your queues
+    string_setting :namespace, nil
+
+    string_setting :group, ''
+
+    # Set of all setting keys
+    ALL_KEYS = @boolean_keys + @number_keys + @string_keys
+
+    def self.initialize(params = {})
+      unless @config
+        @config = default_config
+        define_methods
+        @config.merge!(env_based_config)
+      end
+      @config.merge!(params)
+      @config
+    end
+
+    # Default settings
+    #
+    # @return [Hash]
+    def self.default_config
+      @settings_defaults.merge({
+        mq_exchange_options: {},
         mq_tls_cert: nil,
         mq_tls_key: nil,
-        mq_username: 'guest',
-        mq_password: 'guest',
-        mq_api_host: 'localhost',
-        mq_api_port: 15672,
-        mq_api_ssl: false,
-        heartbeat: 30,
-        # placeholder, allows specifying connection parameters
-        # as a URI.
+        mq_tls_ca_certificates: nil,
         uri: nil,
         log_level: Logger::INFO,
+        client_logger: nil,
         require_paths: [],
-        autoload_rails: true,
         error_handlers: [Hutch::ErrorHandlers::Logger.new],
+        # note that this is not a list, it is a chain of responsibility
+        # that will fall back to "nack unconditionally"
+        error_acknowledgements: [],
+        setup_procs: [],
+        consumer_groups: {},
         tracer: Hutch::Tracers::NullTracer,
         namespace: nil,
-        daemonise: false,
         pidfile: nil,
-        channel_prefetch: 0,
-        # enables publisher confirms, leaves it up to the app
-        # how they are tracked
-        publisher_confirms: false,
-        # like `publisher_confirms` above but also
-        # forces waiting for a confirm for every publish
-        force_publisher_confirms: false,
-        # Heroku needs > 10. MK.
-        connection_timeout: 11,
-        read_timeout: 11,
-        write_timeout: 11,
-        enable_http_api_use: true,
-        # Number of seconds that a running consumer is given
-        # to finish its job when gracefully exiting Hutch, before
-        # it's killed.
-        graceful_exit_timeout: 11,
-        client_logger: nil,
+        serializer: Hutch::Serializers::JSON
+      })
+    end
+
+    # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
+    #
+    # @return [Hash]
+    def self.env_based_config
+      env_keys_configured.each_with_object({}) {|attr, result|
+        value = ENV[key_for(attr)]
 
-        consumer_pool_size: 1,
-      }.merge(params)
+        result[attr] = type_cast(attr, value)
+      }
+    end
+
+    # @return [Array<Symbol>]
+    def self.env_keys_configured
+      ALL_KEYS.each {|attr| check_attr(attr) }
+
+      ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
     end
 
     def self.get(attr)
-      check_attr(attr)
-      user_config[attr]
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym]
+    end
+
+    def self.key_for(attr)
+      key = attr.to_s.gsub('.', '__').upcase
+      "HUTCH_#{key}"
+    end
+
+    def self.is_bool(attr)
+      @boolean_keys.include?(attr)
+    end
+
+    def self.to_bool(value)
+      !(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
+    end
+
+    def self.is_num(attr)
+      @number_keys.include?(attr)
     end
 
     def self.set(attr, value)
-      check_attr(attr)
-      user_config[attr] = value
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym] = type_cast(attr, value)
     end
 
     class << self
@@ -73,17 +241,16 @@ module Hutch
 
     def self.check_attr(attr)
       unless user_config.key?(attr)
-        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+        raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
       end
     end
 
     def self.user_config
-      initialize unless @config
-      @config
+      @config ||= initialize
     end
 
     def self.to_hash
-      self.user_config
+      user_config
     end
 
     def self.load_from_file(file)
@@ -94,28 +261,36 @@ module Hutch
 
     def self.convert_value(attr, value)
       case attr
-      when "tracer"
+      when 'tracer'
         Kernel.const_get(value)
       else
         value
       end
     end
 
-    def self.method_missing(method, *args, &block)
-      attr = method.to_s.sub(/=$/, '').to_sym
-      return super unless user_config.key?(attr)
-
-      if method =~ /=$/
-        set(attr, args.first)
+    def self.type_cast(attr, value)
+      case
+      when is_bool(attr) || value == 'false'
+        to_bool(value)
+      when is_num(attr)
+        value.to_i
       else
-        get(attr)
+        value
       end
     end
+    private_class_method :type_cast
 
-    private
+    def self.define_methods
+      @config.keys.each do |key|
+        define_singleton_method(key) do
+          get(key)
+        end
 
-    def deep_copy(obj)
-      Marshal.load(Marshal.dump(obj))
+        define_singleton_method("#{key}=") do |val|
+          set(key, val)
+        end
+      end
     end
   end
 end
+Hutch::Config.initialize

data/lib/hutch/consumer.rb

@@ -19,7 +19,7 @@ module Hutch
     def requeue!
       broker.requeue(delivery_info.delivery_tag)
     end
-    
+
     def logger
       Hutch::Logging.logger
     end
@@ -29,18 +29,47 @@ module Hutch
       # wants to subscribe to.
       def consume(*routing_keys)
         @routing_keys = self.routing_keys.union(routing_keys)
+        # these are opt-in
+        @queue_mode = nil
+        @queue_type = nil
       end
 
+      attr_reader :queue_mode, :queue_type, :initial_group_size
+
       # Explicitly set the queue name
       def queue_name(name)
         @queue_name = name
       end
 
-      # Allow to specify custom arguments that will be passed when creating the queue.
+      # Explicitly set the queue mode to 'lazy'
+      def lazy_queue
+        @queue_mode = 'lazy'
+      end
+
+      # Explicitly set the queue type to 'classic'
+      def classic_queue
+        @queue_type = 'classic'
+      end
+
+      # Explicitly set the queue type to 'quorum'
+      # @param [Hash] options the options params related to quorum queue
+      # @option options [Integer] :initial_group_size Initial Replication Factor
+      def quorum_queue(options = {})
+        @queue_type = 'quorum'
+        @initial_group_size = options[:initial_group_size]
+      end
+
+      # Configures an optional argument that will be passed when declaring the queue.
+      # Prefer using a policy to this DSL: https://www.rabbitmq.com/parameters.html#policies
       def arguments(arguments = {})
         @arguments = arguments
       end
 
+      # Set custom serializer class, override global value
+      def serializer(name)
+        @serializer = name
+      end
+
       # The RabbitMQ queue name for the consumer. This is derived from the
       # fully-qualified class name. Module separators are replaced with single
       # colons, camelcased class names are converted to snake case.
@@ -53,13 +82,23 @@ module Hutch
 
       # Returns consumer custom arguments.
       def get_arguments
-        @arguments || {}
+        all_arguments = @arguments || {}
+
+        all_arguments['x-queue-mode'] = @queue_mode if @queue_mode
+        all_arguments['x-queue-type'] = @queue_type if @queue_type
+        all_arguments['x-quorum-initial-group-size'] = @initial_group_size if @initial_group_size
+
+        all_arguments
       end
 
       # Accessor for the consumer's routing key.
       def routing_keys
         @routing_keys ||= Set.new
       end
+
+      def get_serializer
+        @serializer
+      end
     end
   end
 end

data/lib/hutch/error_handlers/airbrake.rb

@@ -1,25 +1,53 @@
 require 'hutch/logging'
 require 'airbrake'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Airbrake
-      include Logging
+    class Airbrake < Base
 
-      def handle(message_id, payload, consumer, ex)
-        prefix = "message(#{message_id || '-'}): "
-        logger.error prefix + "Logging event to Airbrake"
-        logger.error prefix + "#{ex.class} - #{ex.message}"
-        ::Airbrake.notify_or_ignore(ex, {
-          :error_class => ex.class.name,
-          :error_message => "#{ ex.class.name }: #{ ex.message }",
-          :backtrace => ex.backtrace,
-          :parameters => {
-            :payload => payload,
-            :consumer => consumer,
-          },
-          :cgi_data => ENV.to_hash,
-        })
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} Logging event to Airbrake"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+
+        if ::Airbrake.respond_to?(:notify_or_ignore)
+          ::Airbrake.notify_or_ignore(ex, {
+            error_class: ex.class.name,
+            error_message: "#{ ex.class.name }: #{ ex.message }",
+            backtrace: ex.backtrace,
+            parameters: {
+              payload: payload,
+              consumer: consumer,
+            },
+            cgi_data: ENV.to_hash,
+          })
+        else
+          ::Airbrake.notify(ex, {
+            payload: payload,
+            consumer: consumer,
+            cgi_data: ENV.to_hash,
+          })
+        end
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Airbrake"
+        logger.error "#{ex.class} - #{ex.message}"
+
+        if ::Airbrake.respond_to?(:notify_or_ignore)
+          ::Airbrake.notify_or_ignore(ex, {
+            error_class: ex.class.name,
+            error_message: "#{ ex.class.name }: #{ ex.message }",
+            backtrace: ex.backtrace,
+            cgi_data: ENV.to_hash,
+          })
+        else
+          ::Airbrake.notify(ex, {
+            cgi_data: ENV.to_hash,
+          })
+        end
       end
     end
   end

data/lib/hutch/error_handlers/base.rb

@@ -0,0 +1,15 @@
+module Hutch
+  module ErrorHandlers
+    class Base
+      include Logging
+
+      def handle(properties, payload, consumer, ex)
+        raise NotImplementedError.new
+      end
+
+      def handle_setup_exception(ex)
+        raise NotImplementedError.new
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/bugsnag.rb

@@ -0,0 +1,30 @@
+require "hutch/logging"
+require "bugsnag"
+require "hutch/error_handlers/base"
+
+module Hutch
+  module ErrorHandlers
+    class Bugsnag < Base
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || "-"}):"
+        logger.error "#{prefix} Logging event to Bugsnag"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+
+        ::Bugsnag.notify(ex) do |report|
+          report.add_tab(:hutch, {
+            payload: payload,
+            consumer: consumer
+          })
+        end
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Bugsnag"
+        logger.error "#{ex.class} - #{ex.message}"
+
+        ::Bugsnag.notify(ex)
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/honeybadger.rb

@@ -1,27 +1,42 @@
 require 'hutch/logging'
 require 'honeybadger'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Honeybadger
-      include Logging
+    # Error handler for the Honeybadger.io service
+    class Honeybadger < Base
 
-      def handle(message_id, payload, consumer, ex)
-        prefix = "message(#{message_id || '-'}): "
-        logger.error prefix + "Logging event to Honeybadger"
-        logger.error prefix + "#{ex.class} - #{ex.message}"
-        ::Honeybadger.notify_or_ignore(
-          :error_class => ex.class.name,
-          :error_message => "#{ ex.class.name }: #{ ex.message }",
-          :backtrace => ex.backtrace,
-          :context => {
-            :message_id => message_id,
-            :consumer => consumer
-          },
-          :parameters => {
-            :payload => payload
-          }
-        )
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} Logging event to Honeybadger"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+        notify_honeybadger(error_class: ex.class.name,
+                           error_message: "#{ex.class.name}: #{ex.message}",
+                           backtrace: ex.backtrace,
+                           context: { message_id: message_id,
+                                      consumer: consumer },
+                           parameters: { payload: payload })
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Honeybadger"
+        logger.error "#{ex.class} - #{ex.message}"
+        notify_honeybadger(error_class: ex.class.name,
+                           error_message: "#{ex.class.name}: #{ex.message}",
+                           backtrace: ex.backtrace)
+      end
+
+      # Wrap API to support 3.0.0+
+      #
+      # @see https://github.com/honeybadger-io/honeybadger-ruby/blob/master/CHANGELOG.md#300---2017-02-06
+      def notify_honeybadger(message)
+        if ::Honeybadger.respond_to?(:notify_or_ignore)
+          ::Honeybadger.notify_or_ignore(message)
+        else
+          ::Honeybadger.notify(message)
+        end
       end
     end
   end

data/lib/hutch/error_handlers/logger.rb

@@ -1,14 +1,20 @@
 require 'hutch/logging'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Logger
-      include Logging
+    class Logger < ErrorHandlers::Base
 
-      def handle(message_id, payload, consumer, ex)
-        prefix = "message(#{message_id || '-'}): "
-        logger.error prefix + "error in consumer '#{consumer}'"
-        logger.error prefix + "#{ex.class} - #{ex.message}"
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} error in consumer '#{consumer}'"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+        logger.error (['backtrace:'] + ex.backtrace).join("\n")
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "#{ex.class} - #{ex.message}"
         logger.error (['backtrace:'] + ex.backtrace).join("\n")
       end
     end

data/lib/hutch/error_handlers/rollbar.rb

@@ -0,0 +1,28 @@
+require 'hutch/logging'
+require 'rollbar'
+require 'hutch/error_handlers/base'
+
+module Hutch
+  module ErrorHandlers
+    class Rollbar < Base
+      def handle(properties, payload, consumer, ex)
+        message_id = properties.message_id
+        prefix = "message(#{message_id || '-'}):"
+        logger.error "#{prefix} Logging event to Rollbar"
+        logger.error "#{prefix} #{ex.class} - #{ex.message}"
+
+        ::Rollbar.error(ex,
+          payload: payload,
+          consumer: consumer
+        )
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Rollbar"
+        logger.error "#{ex.class} - #{ex.message}"
+
+        ::Rollbar.error(ex)
+      end
+    end
+  end
+end