hutch 0.19.0-java

data/lib/hutch/cli.rb

@@ -0,0 +1,205 @@
+require 'optparse'
+
+require 'hutch/version'
+require 'hutch/logging'
+require 'hutch/exceptions'
+require 'hutch/config'
+
+module Hutch
+  class CLI
+    include Logging
+
+    # Run a Hutch worker with the command line interface.
+    def run(argv = ARGV)
+      parse_options(argv)
+
+      ::Process.daemon(true) if Hutch::Config.daemonise
+
+      write_pid if Hutch::Config.pidfile
+
+      Hutch.logger.info "hutch booted with pid #{::Process.pid}"
+
+      if load_app && start_work_loop == :success
+        # If we got here, the worker was shut down nicely
+        Hutch.logger.info 'hutch shut down gracefully'
+        exit 0
+      else
+        Hutch.logger.info 'hutch terminated due to an error'
+        exit 1
+      end
+    end
+
+    def load_app
+      # Try to load a Rails app in the current directory
+      load_rails_app('.') if Hutch::Config.autoload_rails
+      Hutch::Config.require_paths.each do |path|
+        # See if each path is a Rails app. If so, try to load it.
+        next if load_rails_app(path)
+
+        # Given path is not a Rails app, try requiring it as a file
+        logger.info "requiring '#{path}'"
+        begin
+          # Need to add '.' to load path for relative requires
+          $LOAD_PATH << '.'
+          require path
+        rescue LoadError
+          logger.fatal "could not load file '#{path}'"
+          return false
+        ensure
+          # Clean up load path
+          $LOAD_PATH.pop
+        end
+      end
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def load_rails_app(path)
+      # path should point to the app's top level directory
+      if File.directory?(path)
+        # Smells like a Rails app if it's got a config/environment.rb file
+        rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
+        if File.exists?(rails_path)
+          logger.info "found rails project (#{path}), booting app"
+          ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          require rails_path
+          ::Rails.application.eager_load!
+          return true
+        end
+      end
+      false
+    end
+
+    # Kick off the work loop. This method returns when the worker is shut down
+    # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
+    def start_work_loop
+      Hutch.connect
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker.run
+      :success
+    rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      logger.fatal ex.message
+      :error
+    end
+
+    def parse_options(args = ARGV)
+      OptionParser.new do |opts|
+        opts.banner = 'usage: hutch [options]'
+
+        opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
+          Hutch::Config.mq_host = host
+        end
+
+        opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
+          Hutch::Config.mq_port = port
+        end
+
+        opts.on("-t", "--[no-]mq-tls", 'Use TLS for the AMQP connection') do |tls|
+          Hutch::Config.mq_tls = tls
+        end
+
+        opts.on('--mq-tls-cert FILE', 'Certificate  for TLS client verification') do |file|
+          abort "Certificate file '#{file}' not found" unless File.exists?(file)
+          Hutch::Config.mq_tls_cert = file
+        end
+
+        opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
+          abort "Private key file '#{file}' not found" unless File.exists?(file)
+          Hutch::Config.mq_tls_key = file
+        end
+
+        opts.on('--mq-exchange EXCHANGE',
+                'Set the RabbitMQ exchange') do |exchange|
+          Hutch::Config.mq_exchange = exchange
+        end
+
+        opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
+          Hutch::Config.mq_vhost = vhost
+        end
+
+        opts.on('--mq-username USERNAME',
+                'Set the RabbitMQ username') do |username|
+          Hutch::Config.mq_username = username
+        end
+
+        opts.on('--mq-password PASSWORD',
+                'Set the RabbitMQ password') do |password|
+          Hutch::Config.mq_password = password
+        end
+
+        opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
+          Hutch::Config.mq_api_host = host
+        end
+
+        opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
+          Hutch::Config.mq_api_port = port
+        end
+
+        opts.on("-s", "--[no-]mq-api-ssl", 'Use SSL for the RabbitMQ API') do |api_ssl|
+          Hutch::Config.mq_api_ssl = api_ssl
+        end
+
+        opts.on('--config FILE', 'Load Hutch configuration from a file') do |file|
+          begin
+            File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
+          rescue Errno::ENOENT
+            abort "Config file '#{file}' not found"
+          end
+        end
+
+        opts.on('--require PATH', 'Require a Rails app or path') do |path|
+          Hutch::Config.require_paths << path
+        end
+
+        opts.on('--[no-]autoload-rails', 'Require the current rails app directory') do |autoload_rails|
+          Hutch::Config.autoload_rails = autoload_rails
+        end
+
+        opts.on('-q', '--quiet', 'Quiet logging') do
+          Hutch::Config.log_level = Logger::WARN
+        end
+
+        opts.on('-v', '--verbose', 'Verbose logging') do
+          Hutch::Config.log_level = Logger::DEBUG
+        end
+
+        opts.on('--namespace NAMESPACE', 'Queue namespace') do |namespace|
+          Hutch::Config.namespace = namespace
+        end
+
+        opts.on('-d', '--daemonise', 'Daemonise') do |daemonise|
+          Hutch::Config.daemonise = daemonise
+        end
+
+        opts.on('--pidfile PIDFILE', 'Pidfile') do |pidfile|
+          Hutch::Config.pidfile = pidfile
+        end
+
+        opts.on('--version', 'Print the version and exit') do
+          puts "hutch v#{VERSION}"
+          exit 0
+        end
+
+        opts.on('-h', '--help', 'Show this message and exit') do
+          puts opts
+          exit 0
+        end
+      end.parse!(args)
+    end
+
+    def write_pid
+      pidfile = File.expand_path(Hutch::Config.pidfile)
+      Hutch.logger.info "writing pid in #{pidfile}"
+      File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
+    end
+
+  end
+end

data/lib/hutch/config.rb

@@ -0,0 +1,125 @@
+require 'hutch/error_handlers/logger'
+require 'erb'
+require 'logger'
+
+module Hutch
+  class UnknownAttributeError < StandardError; end
+
+  module Config
+    require 'yaml'
+
+    def self.initialize(params={})
+      @config = {
+        mq_host: 'localhost',
+        mq_port: 5672,
+        mq_exchange: 'hutch',  # TODO: should this be required?
+        mq_vhost: '/',
+        mq_tls: false,
+        mq_tls_cert: nil,
+        mq_tls_key: nil,
+        mq_tls_ca_certificates: nil,
+        mq_verify_peer: true,
+        mq_username: 'guest',
+        mq_password: 'guest',
+        mq_api_host: 'localhost',
+        mq_api_port: 15672,
+        mq_api_ssl: false,
+        heartbeat: 30,
+        # placeholder, allows specifying connection parameters
+        # as a URI.
+        uri: nil,
+        log_level: Logger::INFO,
+        require_paths: [],
+        autoload_rails: true,
+        error_handlers: [Hutch::ErrorHandlers::Logger.new],
+        tracer: Hutch::Tracers::NullTracer,
+        namespace: nil,
+        daemonise: false,
+        pidfile: nil,
+        channel_prefetch: 0,
+        # enables publisher confirms, leaves it up to the app
+        # how they are tracked
+        publisher_confirms: false,
+        # like `publisher_confirms` above but also
+        # forces waiting for a confirm for every publish
+        force_publisher_confirms: false,
+        # Heroku needs > 10. MK.
+        connection_timeout: 11,
+        read_timeout: 11,
+        write_timeout: 11,
+        enable_http_api_use: true,
+        # Number of seconds that a running consumer is given
+        # to finish its job when gracefully exiting Hutch, before
+        # it's killed.
+        graceful_exit_timeout: 11,
+        client_logger: nil,
+
+        consumer_pool_size: 1,
+
+        serializer: Hutch::Serializers::JSON,
+      }.merge(params)
+    end
+
+    def self.get(attr)
+      check_attr(attr)
+      user_config[attr]
+    end
+
+    def self.set(attr, value)
+      check_attr(attr)
+      user_config[attr] = value
+    end
+
+    class << self
+      alias_method :[],  :get
+      alias_method :[]=, :set
+    end
+
+    def self.check_attr(attr)
+      unless user_config.key?(attr)
+        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+      end
+    end
+
+    def self.user_config
+      initialize unless @config
+      @config
+    end
+
+    def self.to_hash
+      self.user_config
+    end
+
+    def self.load_from_file(file)
+      YAML.load(ERB.new(File.read(file)).result).each do |attr, value|
+        Hutch::Config.send("#{attr}=", convert_value(attr, value))
+      end
+    end
+
+    def self.convert_value(attr, value)
+      case attr
+      when "tracer"
+        Kernel.const_get(value)
+      else
+        value
+      end
+    end
+
+    def self.method_missing(method, *args, &block)
+      attr = method.to_s.sub(/=$/, '').to_sym
+      return super unless user_config.key?(attr)
+
+      if method =~ /=$/
+        set(attr, args.first)
+      else
+        get(attr)
+      end
+    end
+
+    private
+
+    def deep_copy(obj)
+      Marshal.load(Marshal.dump(obj))
+    end
+  end
+end

data/lib/hutch/consumer.rb

@@ -0,0 +1,75 @@
+require 'set'
+
+module Hutch
+  # Include this module in a class to register it as a consumer. Consumers
+  # gain a class method called `consume`, which should be used to register
+  # the routing keys a consumer is interested in.
+  module Consumer
+    attr_accessor :broker, :delivery_info
+
+    def self.included(base)
+      base.extend(ClassMethods)
+      Hutch.register_consumer(base)
+    end
+
+    def reject!
+      broker.reject(delivery_info.delivery_tag)
+    end
+
+    def requeue!
+      broker.requeue(delivery_info.delivery_tag)
+    end
+
+    def logger
+      Hutch::Logging.logger
+    end
+
+    module ClassMethods
+      # Add one or more routing keys to the set of routing keys the consumer
+      # wants to subscribe to.
+      def consume(*routing_keys)
+        @routing_keys = self.routing_keys.union(routing_keys)
+      end
+
+      # Explicitly set the queue name
+      def queue_name(name)
+        @queue_name = name
+      end
+
+      # Allow to specify custom arguments that will be passed when creating the queue.
+      def arguments(arguments = {})
+        @arguments = arguments
+      end
+
+      # Set custom serializer class, override global value
+      def serializer(name)
+        @serializer = name
+      end
+
+      # The RabbitMQ queue name for the consumer. This is derived from the
+      # fully-qualified class name. Module separators are replaced with single
+      # colons, camelcased class names are converted to snake case.
+      def get_queue_name
+        return @queue_name unless @queue_name.nil?
+        queue_name = self.name.gsub(/::/, ':')
+        queue_name.gsub!(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }
+        queue_name.downcase
+      end
+
+      # Returns consumer custom arguments.
+      def get_arguments
+        @arguments || {}
+      end
+
+      # Accessor for the consumer's routing key.
+      def routing_keys
+        @routing_keys ||= Set.new
+      end
+
+      def get_serializer
+        @serializer
+      end
+    end
+  end
+end
+

data/lib/hutch/error_handlers.rb

@@ -0,0 +1,8 @@
+module Hutch
+  module ErrorHandlers
+    autoload :Logger,      'hutch/error_handlers/logger'
+    autoload :Sentry,      'hutch/error_handlers/sentry'
+    autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
+    autoload :Airbrake,    'hutch/error_handlers/airbrake'
+  end
+end

data/lib/hutch/error_handlers/airbrake.rb

@@ -0,0 +1,26 @@
+require 'hutch/logging'
+require 'airbrake'
+
+module Hutch
+  module ErrorHandlers
+    class Airbrake
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Airbrake"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        ::Airbrake.notify_or_ignore(ex, {
+          :error_class => ex.class.name,
+          :error_message => "#{ ex.class.name }: #{ ex.message }",
+          :backtrace => ex.backtrace,
+          :parameters => {
+            :payload => payload,
+            :consumer => consumer,
+          },
+          :cgi_data => ENV.to_hash,
+        })
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/honeybadger.rb

@@ -0,0 +1,28 @@
+require 'hutch/logging'
+require 'honeybadger'
+
+module Hutch
+  module ErrorHandlers
+    class Honeybadger
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Honeybadger"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        ::Honeybadger.notify_or_ignore(
+          :error_class => ex.class.name,
+          :error_message => "#{ ex.class.name }: #{ ex.message }",
+          :backtrace => ex.backtrace,
+          :context => {
+            :message_id => message_id,
+            :consumer => consumer
+          },
+          :parameters => {
+            :payload => payload
+          }
+        )
+      end
+    end
+  end
+end