hutch 0.21.0-java → 0.25.0-java

data/lib/hutch/cli.rb

@@ -1,9 +1,9 @@
 require 'optparse'
 
-require 'hutch/version'
 require 'hutch/logging'
-require 'hutch/exceptions'
 require 'hutch/config'
+require 'hutch/version'
+require 'hutch/exceptions'
 
 module Hutch
   class CLI
@@ -11,9 +11,10 @@ module Hutch
 
     # Run a Hutch worker with the command line interface.
     def run(argv = ARGV)
+      Hutch::Config.initialize
       parse_options(argv)
 
-      ::Process.daemon(true) if Hutch::Config.daemonise
+      daemonise_process
 
       write_pid if Hutch::Config.pidfile
 
@@ -71,8 +72,8 @@ module Hutch
         end
         rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
         if is_rails_app && File.exist?(rails_path)
-          logger.info "found rails project (#{path}), booting app"
           ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
           require rails_path
           ::Rails.application.eager_load!
           return true
@@ -85,10 +86,13 @@ module Hutch
     # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
     def start_work_loop
       Hutch.connect
-      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
       @worker.run
       :success
     rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle_setup_exception(ex)
+      end
       logger.fatal ex.message
       :error
     end
@@ -109,14 +113,16 @@ module Hutch
           Hutch::Config.mq_tls = tls
         end
 
-        opts.on('--mq-tls-cert FILE', 'Certificate  for TLS client verification') do |file|
-          abort "Certificate file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_cert = file
+        opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
+          abort_without_file(file, 'Certificate file') do
+            Hutch::Config.mq_tls_cert = file
+          end
         end
 
         opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
-          abort "Private key file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_key = file
+          abort_without_file(file, 'Private key file') do
+            Hutch::Config.mq_tls_key = file
+          end
         end
 
         opts.on('--mq-exchange EXCHANGE',
@@ -154,7 +160,7 @@ module Hutch
           begin
             File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
           rescue Errno::ENOENT
-            abort "Config file '#{file}' not found"
+            abort_with_message("Config file '#{file}' not found")
           end
         end
 
@@ -186,6 +192,10 @@ module Hutch
           Hutch::Config.pidfile = pidfile
         end
 
+        opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
+          Hutch::Config.group = group
+        end
+
         opts.on('--version', 'Print the version and exit') do
           puts "hutch v#{VERSION}"
           exit 0
@@ -204,5 +214,26 @@ module Hutch
       File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
     end
 
+    private
+
+    def daemonise_process
+      return unless Hutch::Config.daemonise
+      if defined?(JRUBY_VERSION)
+        Hutch.logger.warn "JRuby ignores the --daemonise option"
+        return
+      end
+
+      ::Process.daemon(true)
+    end
+
+    def abort_without_file(file, file_description, &block)
+      abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
+
+      yield
+    end
+
+    def abort_with_message(message)
+      abort message
+    end
   end
 end

data/lib/hutch/config.rb

@@ -1,77 +1,232 @@
 require 'hutch/error_handlers/logger'
+require 'hutch/tracers'
+require 'hutch/serializers/json'
 require 'erb'
 require 'logger'
 
 module Hutch
   class UnknownAttributeError < StandardError; end
 
+  # Configuration settings, available everywhere
+  #
+  # There are defaults, which can be overridden by ENV variables prefixed by
+  # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
+  # method.
+  #
+  # @example Configuring on the command-line
+  #   HUTCH_PUBLISHER_CONFIRMS=false hutch
   module Config
     require 'yaml'
+    @string_keys = Set.new
+    @number_keys = Set.new
+    @boolean_keys = Set.new
+    @settings_defaults = {}
+
+    # Define a String user setting
+    # @!visibility private
+    def self.string_setting(name, default_value)
+      @string_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Number user setting
+    # @!visibility private
+    def self.number_setting(name, default_value)
+      @number_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Boolean user setting
+    # @!visibility private
+    def self.boolean_setting(name, default_value)
+      @boolean_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # RabbitMQ hostname
+    string_setting :mq_host, '127.0.0.1'
+
+    # RabbitMQ Exchange to use for publishing
+    string_setting :mq_exchange, 'hutch'
+
+    # RabbitMQ vhost to use
+    string_setting :mq_vhost, '/'
+
+    # RabbitMQ username to use.
+    #
+    # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
+    string_setting :mq_username, 'guest'
+
+    # RabbitMQ password
+    string_setting :mq_password, 'guest'
+
+    # RabbitMQ URI (takes precedence over MQ username, password, host, port and vhost settings)
+    string_setting :uri, nil
+
+    # RabbitMQ HTTP API hostname
+    string_setting :mq_api_host, '127.0.0.1'
+
+    # RabbitMQ port
+    number_setting :mq_port, 5672
+
+    # RabbitMQ HTTP API port
+    number_setting :mq_api_port, 15672
+
+    # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
+    number_setting :heartbeat, 30
+
+    # The <tt>basic.qos</tt> prefetch value to use.
+    #
+    # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
+    number_setting :channel_prefetch, 0
+
+    # Bunny's socket open timeout
+    number_setting :connection_timeout, 11
+
+    # Bunny's socket read timeout
+    number_setting :read_timeout, 11
+
+    # Bunny's socket write timeout
+    number_setting :write_timeout, 11
+
+    # FIXME: DOCUMENT THIS
+    number_setting :graceful_exit_timeout, 11
+
+    # Bunny consumer work pool size
+    number_setting :consumer_pool_size, 1
+
+    # Should TLS be used?
+    boolean_setting :mq_tls, false
+
+    # Should SSL certificate be verified?
+    boolean_setting :mq_verify_peer, true
+
+    # Should SSL be used for the RabbitMQ API?
+    boolean_setting :mq_api_ssl, false
+
+    # Should the current Rails app directory be required?
+    boolean_setting :autoload_rails, true
+
+    # Should the Hutch runner process daemonise?
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :daemonise, false
+
+    # Should RabbitMQ publisher confirms be enabled?
+    #
+    # Leaves it up to the app how they are tracked
+    # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
+    boolean_setting :publisher_confirms, false
+
+    # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
+    # every publish.
+    #
+    # **This is the safest option which also offers the
+    # lowest throughput**.
+    boolean_setting :force_publisher_confirms, false
+
+    # Should the RabbitMQ HTTP API be used?
+    boolean_setting :enable_http_api_use, true
+
+    # Should Bunny's consumer work pool threads abort on exception.
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :consumer_pool_abort_on_exception, false
+
+    # Prefix displayed on the consumers tags.
+    string_setting :consumer_tag_prefix, 'hutch'
+
+    string_setting :group, ''
+
+    # Set of all setting keys
+    ALL_KEYS = @boolean_keys + @number_keys + @string_keys
 
     def self.initialize(params = {})
-      @config = {
-        mq_host: 'localhost',
-        mq_port: 5672,
-        mq_exchange: 'hutch',  # TODO: should this be required?
+      unless @config
+        @config = default_config
+        define_methods
+        @config.merge!(env_based_config)
+      end
+      @config.merge!(params)
+      @config
+    end
+
+    # Default settings
+    #
+    # @return [Hash]
+    def self.default_config
+      @settings_defaults.merge({
         mq_exchange_options: {},
-        mq_vhost: '/',
-        mq_tls: false,
         mq_tls_cert: nil,
         mq_tls_key: nil,
         mq_tls_ca_certificates: nil,
-        mq_verify_peer: true,
-        mq_username: 'guest',
-        mq_password: 'guest',
-        mq_api_host: 'localhost',
-        mq_api_port: 15672,
-        mq_api_ssl: false,
-        heartbeat: 30,
-        # placeholder, allows specifying connection parameters
-        # as a URI.
         uri: nil,
         log_level: Logger::INFO,
+        client_logger: nil,
         require_paths: [],
-        autoload_rails: true,
         error_handlers: [Hutch::ErrorHandlers::Logger.new],
         # note that this is not a list, it is a chain of responsibility
         # that will fall back to "nack unconditionally"
         error_acknowledgements: [],
+        setup_procs: [],
+        consumer_groups: {},
         tracer: Hutch::Tracers::NullTracer,
         namespace: nil,
-        daemonise: false,
         pidfile: nil,
-        channel_prefetch: 0,
-        # enables publisher confirms, leaves it up to the app
-        # how they are tracked
-        publisher_confirms: false,
-        # like `publisher_confirms` above but also
-        # forces waiting for a confirm for every publish
-        force_publisher_confirms: false,
-        # Heroku needs > 10. MK.
-        connection_timeout: 11,
-        read_timeout: 11,
-        write_timeout: 11,
-        enable_http_api_use: true,
-        # Number of seconds that a running consumer is given
-        # to finish its job when gracefully exiting Hutch, before
-        # it's killed.
-        graceful_exit_timeout: 11,
-        client_logger: nil,
+        serializer: Hutch::Serializers::JSON
+      })
+    end
+
+    # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
+    #
+    # @return [Hash]
+    def self.env_based_config
+      env_keys_configured.each_with_object({}) {|attr, result|
+        value = ENV[key_for(attr)]
+
+        case
+        when is_bool(attr) || value == 'false'
+          result[attr] = to_bool(value)
+        when is_num(attr)
+          result[attr] = value.to_i
+        else
+          result[attr] = value
+        end
+      }
+    end
 
-        consumer_pool_size: 1,
+    # @return [Array<Symbol>]
+    def self.env_keys_configured
+      ALL_KEYS.each {|attr| check_attr(attr) }
 
-        serializer: Hutch::Serializers::JSON,
-      }.merge(params)
+      ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
     end
 
     def self.get(attr)
-      check_attr(attr)
-      user_config[attr]
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym]
+    end
+
+    def self.key_for(attr)
+      key = attr.to_s.gsub('.', '__').upcase
+      "HUTCH_#{key}"
+    end
+
+    def self.is_bool(attr)
+      @boolean_keys.include?(attr)
+    end
+
+    def self.to_bool(value)
+      !(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
+    end
+
+    def self.is_num(attr)
+      @number_keys.include?(attr)
     end
 
     def self.set(attr, value)
-      check_attr(attr)
-      user_config[attr] = value
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym] = value
     end
 
     class << self
@@ -81,17 +236,16 @@ module Hutch
 
     def self.check_attr(attr)
       unless user_config.key?(attr)
-        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+        raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
       end
     end
 
     def self.user_config
-      initialize unless @config
-      @config
+      @config ||= initialize
     end
 
     def self.to_hash
-      self.user_config
+      user_config
     end
 
     def self.load_from_file(file)
@@ -102,28 +256,24 @@ module Hutch
 
     def self.convert_value(attr, value)
       case attr
-      when "tracer"
+      when 'tracer'
         Kernel.const_get(value)
       else
         value
       end
     end
 
-    def self.method_missing(method, *args, &block)
-      attr = method.to_s.sub(/=$/, '').to_sym
-      return super unless user_config.key?(attr)
+    def self.define_methods
+      @config.keys.each do |key|
+        define_singleton_method(key) do
+          get(key)
+        end
 
-      if method =~ /=$/
-        set(attr, args.first)
-      else
-        get(attr)
+        define_singleton_method("#{key}=") do |val|
+          set(key, val)
+        end
       end
     end
-
-    private
-
-    def deep_copy(obj)
-      Marshal.load(Marshal.dump(obj))
-    end
   end
 end
+Hutch::Config.initialize

data/lib/hutch/error_handlers.rb

@@ -4,5 +4,6 @@ module Hutch
     autoload :Sentry,      'hutch/error_handlers/sentry'
     autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
     autoload :Airbrake,    'hutch/error_handlers/airbrake'
+    autoload :Opbeat,      'hutch/error_handlers/opbeat'
   end
 end