hutch 0.21.0 → 0.22.1

data/lib/hutch/cli.rb

@@ -1,9 +1,9 @@
 require 'optparse'
 
-require 'hutch/version'
 require 'hutch/logging'
-require 'hutch/exceptions'
 require 'hutch/config'
+require 'hutch/version'
+require 'hutch/exceptions'
 
 module Hutch
   class CLI
@@ -11,9 +11,10 @@ module Hutch
 
     # Run a Hutch worker with the command line interface.
     def run(argv = ARGV)
+      Hutch::Config.initialize
       parse_options(argv)
 
-      ::Process.daemon(true) if Hutch::Config.daemonise
+      daemonise_process
 
       write_pid if Hutch::Config.pidfile
 
@@ -85,7 +86,7 @@ module Hutch
     # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
     def start_work_loop
       Hutch.connect
-      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
       @worker.run
       :success
     rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
@@ -109,14 +110,16 @@ module Hutch
           Hutch::Config.mq_tls = tls
         end
 
-        opts.on('--mq-tls-cert FILE', 'Certificate  for TLS client verification') do |file|
-          abort "Certificate file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_cert = file
+        opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
+          abort_without_file(file, 'Certificate file') do
+            Hutch::Config.mq_tls_cert = file
+          end
         end
 
         opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
-          abort "Private key file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_key = file
+          abort_without_file(file, 'Private key file') do
+            Hutch::Config.mq_tls_key = file
+          end
         end
 
         opts.on('--mq-exchange EXCHANGE',
@@ -154,7 +157,7 @@ module Hutch
           begin
             File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
           rescue Errno::ENOENT
-            abort "Config file '#{file}' not found"
+            abort_with_message("Config file '#{file}' not found")
           end
         end
 
@@ -204,5 +207,26 @@ module Hutch
       File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
     end
 
+    private
+
+    def daemonise_process
+      return unless Hutch::Config.daemonise
+      if defined?(JRUBY_VERSION)
+        Hutch.logger.warn "JRuby ignores the --daemonise option"
+        return
+      end
+
+      ::Process.daemon(true)
+    end
+
+    def abort_without_file(file, file_description, &block)
+      abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
+
+      yield
+    end
+
+    def abort_with_message(message)
+      abort message
+    end
   end
 end

data/lib/hutch/config.rb

@@ -1,67 +1,193 @@
 require 'hutch/error_handlers/logger'
+require 'hutch/tracers'
+require 'hutch/serializers/json'
 require 'erb'
 require 'logger'
 
 module Hutch
   class UnknownAttributeError < StandardError; end
 
+  # Configuration settings, available everywhere
+  #
+  # There are defaults, which can be overridden by ENV variables prefixed by
+  # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
+  # method.
+  #
+  # @example Configuring on the command-line
+  #   HUTCH_PUBLISHER_CONFIRMS=false hutch
   module Config
     require 'yaml'
+    @string_keys = Set.new
+    @number_keys = Set.new
+    @boolean_keys = Set.new
+    @settings_defaults = {}
+
+    # Define a String user setting
+    # @!visibility private
+    def self.string_setting(name, default_value)
+      @string_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Number user setting
+    # @!visibility private
+    def self.number_setting(name, default_value)
+      @number_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Boolean user setting
+    # @!visibility private
+    def self.boolean_setting(name, default_value)
+      @boolean_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # RabbitMQ hostname
+    string_setting :mq_host, '127.0.0.1'
+
+    # RabbitMQ Exchange to use for publishing
+    string_setting :mq_exchange, 'hutch'
+
+    # RabbitMQ vhost to use
+    string_setting :mq_vhost, '/'
+
+    # RabbitMQ username to use.
+    #
+    # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
+    string_setting :mq_username, 'guest'
+
+    # RabbitMQ password
+    string_setting :mq_password, 'guest'
+
+    # RabbitMQ HTTP API hostname
+    string_setting :mq_api_host, '127.0.0.1'
+
+    # RabbitMQ port
+    number_setting :mq_port, 5672
+
+    # RabbitMQ HTTP API port
+    number_setting :mq_api_port, 15672
+
+    # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
+    number_setting :heartbeat, 30
+
+    # The <tt>basic.qos</tt> prefetch value to use.
+    #
+    # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
+    number_setting :channel_prefetch, 0
+
+    # Bunny's socket open timeout
+    number_setting :connection_timeout, 11
+
+    # Bunny's socket read timeout
+    number_setting :read_timeout, 11
+
+    # Bunny's socket write timeout
+    number_setting :write_timeout, 11
+
+    # FIXME: DOCUMENT THIS
+    number_setting :graceful_exit_timeout, 11
+
+    # Bunny consumer work pool size
+    number_setting :consumer_pool_size, 1
+
+    # Should TLS be used?
+    boolean_setting :mq_tls, false
+
+    # Should SSL certificate be verified?
+    boolean_setting :mq_verify_peer, true
+
+    # Should SSL be used for the RabbitMQ API?
+    boolean_setting :mq_api_ssl, false
+
+    # Should the current Rails app directory be required?
+    boolean_setting :autoload_rails, true
+
+    # Should the Hutch runner process daemonise?
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :daemonise, false
+
+    # Should RabbitMQ publisher confirms be enabled?
+    #
+    # Leaves it up to the app how they are tracked
+    # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
+    boolean_setting :publisher_confirms, false
+
+    # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
+    # every publish.
+    #
+    # **This is the safest option which also offers the
+    # lowest throughput**.
+    boolean_setting :force_publisher_confirms, false
+
+    # Should the RabbitMQ HTTP API be used?
+    boolean_setting :enable_http_api_use, true
+
+    # Should Bunny's consumer work pool threads abort on exception.
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :consumer_pool_abort_on_exception, false
+
+    # Set of all setting keys
+    ALL_KEYS = @boolean_keys + @number_keys + @string_keys
 
     def self.initialize(params = {})
-      @config = {
-        mq_host: 'localhost',
-        mq_port: 5672,
-        mq_exchange: 'hutch',  # TODO: should this be required?
+      @config = default_config
+      @config.merge!(env_based_config).merge!(params)
+      define_methods
+      @config
+    end
+
+    # Default settings
+    #
+    # @return [Hash]
+    def self.default_config
+      @settings_defaults.merge({
         mq_exchange_options: {},
-        mq_vhost: '/',
-        mq_tls: false,
         mq_tls_cert: nil,
         mq_tls_key: nil,
         mq_tls_ca_certificates: nil,
-        mq_verify_peer: true,
-        mq_username: 'guest',
-        mq_password: 'guest',
-        mq_api_host: 'localhost',
-        mq_api_port: 15672,
-        mq_api_ssl: false,
-        heartbeat: 30,
-        # placeholder, allows specifying connection parameters
-        # as a URI.
         uri: nil,
         log_level: Logger::INFO,
+        client_logger: nil,
         require_paths: [],
-        autoload_rails: true,
         error_handlers: [Hutch::ErrorHandlers::Logger.new],
         # note that this is not a list, it is a chain of responsibility
         # that will fall back to "nack unconditionally"
         error_acknowledgements: [],
+        setup_procs: [],
         tracer: Hutch::Tracers::NullTracer,
         namespace: nil,
-        daemonise: false,
         pidfile: nil,
-        channel_prefetch: 0,
-        # enables publisher confirms, leaves it up to the app
-        # how they are tracked
-        publisher_confirms: false,
-        # like `publisher_confirms` above but also
-        # forces waiting for a confirm for every publish
-        force_publisher_confirms: false,
-        # Heroku needs > 10. MK.
-        connection_timeout: 11,
-        read_timeout: 11,
-        write_timeout: 11,
-        enable_http_api_use: true,
-        # Number of seconds that a running consumer is given
-        # to finish its job when gracefully exiting Hutch, before
-        # it's killed.
-        graceful_exit_timeout: 11,
-        client_logger: nil,
+        serializer: Hutch::Serializers::JSON
+      })
+    end
+
+    # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
+    #
+    # @return [Hash]
+    def self.env_based_config
+      env_keys_configured.each_with_object({}) {|attr, result|
+        value = ENV[key_for(attr)]
+
+        case
+        when is_bool(attr) || value == 'false'
+          result[attr] = to_bool(value)
+        when is_num(attr)
+          result[attr] = value.to_i
+        else
+          result[attr] = value
+        end
+      }
+    end
 
-        consumer_pool_size: 1,
+    # @return [Array<Symbol>]
+    def self.env_keys_configured
+      ALL_KEYS.each {|attr| check_attr(attr) }
 
-        serializer: Hutch::Serializers::JSON,
-      }.merge(params)
+      ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
     end
 
     def self.get(attr)
@@ -69,6 +195,23 @@ module Hutch
       user_config[attr]
     end
 
+    def self.key_for(attr)
+      key = attr.to_s.gsub('.', '__').upcase
+      "HUTCH_#{key}"
+    end
+
+    def self.is_bool(attr)
+      @boolean_keys.include?(attr)
+    end
+
+    def self.to_bool(value)
+      !(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
+    end
+
+    def self.is_num(attr)
+      @number_keys.include?(attr)
+    end
+
     def self.set(attr, value)
       check_attr(attr)
       user_config[attr] = value
@@ -81,17 +224,16 @@ module Hutch
 
     def self.check_attr(attr)
       unless user_config.key?(attr)
-        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+        raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
       end
     end
 
     def self.user_config
-      initialize unless @config
-      @config
+      @config ||= initialize
     end
 
     def self.to_hash
-      self.user_config
+      user_config
     end
 
     def self.load_from_file(file)
@@ -102,28 +244,23 @@ module Hutch
 
     def self.convert_value(attr, value)
       case attr
-      when "tracer"
+      when 'tracer'
         Kernel.const_get(value)
       else
         value
       end
     end
 
-    def self.method_missing(method, *args, &block)
-      attr = method.to_s.sub(/=$/, '').to_sym
-      return super unless user_config.key?(attr)
+    def self.define_methods
+      @config.keys.each do |key|
+        define_singleton_method(key) do
+          get(key)
+        end
 
-      if method =~ /=$/
-        set(attr, args.first)
-      else
-        get(attr)
+        define_singleton_method("#{key}=") do |val|
+          set(key, val)
+        end
       end
     end
-
-    private
-
-    def deep_copy(obj)
-      Marshal.load(Marshal.dump(obj))
-    end
   end
 end

data/lib/hutch/error_handlers/sentry.rb

@@ -16,7 +16,7 @@ module Hutch
         prefix = "message(#{message_id || '-'}): "
         logger.error prefix + "Logging event to Sentry"
         logger.error prefix + "#{ex.class} - #{ex.message}"
-        Raven.capture_exception(ex)
+        Raven.capture_exception(ex, extra: { payload: payload })
       end
     end
   end

data/lib/hutch/logging.rb

@@ -11,6 +11,7 @@ module Hutch
 
     def self.setup_logger(target = $stdout)
       require 'hutch/config'
+      Hutch::Config.initialize
       @logger = Logger.new(target)
       @logger.level = Hutch::Config.log_level
       @logger.formatter = HutchFormatter.new

data/lib/hutch/publisher.rb

@@ -0,0 +1,75 @@
+require 'securerandom'
+require 'hutch/logging'
+require 'hutch/exceptions'
+
+module Hutch
+  class Publisher
+    include Logging
+    attr_reader :connection, :channel, :exchange, :config
+
+    def initialize(connection, channel, exchange, config = Hutch::Config)
+      @connection = connection
+      @channel    = channel
+      @exchange   = exchange
+      @config     = config
+    end
+
+    def publish(routing_key, message, properties = {}, options = {})
+      ensure_connection!(routing_key, message)
+
+      serializer = options[:serializer] || config[:serializer]
+
+      non_overridable_properties = {
+        routing_key:  routing_key,
+        timestamp:    connection.current_timestamp,
+        content_type: serializer.content_type,
+      }
+      properties[:message_id]   ||= generate_id
+
+      payload = serializer.encode(message)
+
+      log_publication(serializer, payload, routing_key)
+
+      response = exchange.publish(payload, {persistent: true}.
+        merge(properties).
+        merge(global_properties).
+        merge(non_overridable_properties))
+
+      channel.wait_for_confirms if config[:force_publisher_confirms]
+      response
+    end
+
+    private
+
+    def log_publication(serializer, payload, routing_key)
+      logger.info {
+        spec =
+          if serializer.binary?
+            "#{payload.bytesize} bytes message"
+          else
+            "message '#{payload}'"
+          end
+        "publishing #{spec} to #{routing_key}"
+      }
+    end
+
+    def raise_publish_error(reason, routing_key, message)
+      msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
+      logger.error(msg)
+      raise PublishError, msg
+    end
+
+    def ensure_connection!(routing_key, message)
+      raise_publish_error('no connection to broker', routing_key, message) unless connection
+      raise_publish_error('connection is closed', routing_key, message) unless connection.open?
+    end
+
+    def generate_id
+      SecureRandom.uuid
+    end
+
+    def global_properties
+      Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
+    end
+  end
+end