hutch-java 1.3.0-java

data/lib/hutch/cli.rb

@@ -0,0 +1,246 @@
+require 'optparse'
+
+require 'hutch/logging'
+require 'hutch/config'
+require 'hutch/version'
+require 'hutch/exceptions'
+
+module Hutch
+  class CLI
+    include Logging
+
+    # Run a Hutch worker with the command line interface.
+    def run(argv = ARGV)
+      Hutch::Config.initialize
+      parse_options(argv)
+
+      daemonise_process
+
+      write_pid if Hutch::Config.pidfile
+
+      Hutch.logger.info "hutch booted with pid #{::Process.pid}"
+
+      if load_app && start_work_loop == :success
+        # If we got here, the worker was shut down nicely
+        Hutch.logger.info 'hutch shut down gracefully'
+        exit 0
+      else
+        Hutch.logger.info 'hutch terminated due to an error'
+        exit 1
+      end
+    end
+
+    def load_app
+      # Try to load a Rails app in the current directory
+      load_rails_app('.') if Hutch::Config.autoload_rails
+      set_up_code_paths!
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::SentryRaven.new
+      end
+      if defined?(Sentry)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def set_up_code_paths!
+      Hutch::Config.require_paths.each do |path|
+        # See if each path is a Rails app. If so, try to load it.
+        next if load_rails_app(path)
+
+        # Given path is not a Rails app, try requiring it as a file
+        logger.info "requiring '#{path}'"
+        begin
+          # Need to add '.' to load path for relative requires
+          $LOAD_PATH << '.'
+          require path
+        rescue LoadError => e
+          logger.fatal "could not load file '#{path}': #{e}"
+          return false
+        ensure
+          # Clean up load path
+          $LOAD_PATH.delete('.')
+        end
+      end
+    end
+
+    def load_rails_app(path)
+      # path should point to the app's top level directory
+      if File.directory?(path)
+        # Smells like a Rails app if it's got a script/rails or bin/rails file
+        is_rails_app = ['script/rails', 'bin/rails'].any? do |file|
+          File.exist?(File.expand_path(File.join(path, file)))
+        end
+        rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
+        if is_rails_app && File.exist?(rails_path)
+          ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
+          require rails_path
+          ::Rails.application.eager_load!
+          return true
+        end
+      end
+      false
+    end
+
+    # Kick off the work loop. This method returns when the worker is shut down
+    # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
+    def start_work_loop
+      Hutch.connect
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
+      @worker.run
+      :success
+    rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle_setup_exception(ex)
+      end
+      logger.fatal ex.message
+      :error
+    end
+
+    def parse_options(args = ARGV)
+      OptionParser.new do |opts|
+        opts.banner = 'usage: hutch [options]'
+
+        opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
+          Hutch::Config.mq_host = host
+        end
+
+        opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
+          Hutch::Config.mq_port = port
+        end
+
+        opts.on("-t", "--[no-]mq-tls", 'Use TLS for the AMQP connection') do |tls|
+          Hutch::Config.mq_tls = tls
+        end
+
+        opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
+          abort_without_file(file, 'Certificate file') do
+            Hutch::Config.mq_tls_cert = file
+          end
+        end
+
+        opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
+          abort_without_file(file, 'Private key file') do
+            Hutch::Config.mq_tls_key = file
+          end
+        end
+
+        opts.on('--mq-exchange EXCHANGE',
+                'Set the RabbitMQ exchange') do |exchange|
+          Hutch::Config.mq_exchange = exchange
+        end
+
+        opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
+          Hutch::Config.mq_vhost = vhost
+        end
+
+        opts.on('--mq-username USERNAME',
+                'Set the RabbitMQ username') do |username|
+          Hutch::Config.mq_username = username
+        end
+
+        opts.on('--mq-password PASSWORD',
+                'Set the RabbitMQ password') do |password|
+          Hutch::Config.mq_password = password
+        end
+
+        opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
+          Hutch::Config.mq_api_host = host
+        end
+
+        opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
+          Hutch::Config.mq_api_port = port
+        end
+
+        opts.on("-s", "--[no-]mq-api-ssl", 'Use SSL for the RabbitMQ API') do |api_ssl|
+          Hutch::Config.mq_api_ssl = api_ssl
+        end
+
+        opts.on('--config FILE', 'Load Hutch configuration from a file') do |file|
+          begin
+            File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
+          rescue Errno::ENOENT
+            abort_with_message("Config file '#{file}' not found")
+          end
+        end
+
+        opts.on('--require PATH', 'Require a Rails app or path') do |path|
+          Hutch::Config.require_paths << path
+        end
+
+        opts.on('--[no-]autoload-rails', 'Require the current rails app directory') do |autoload_rails|
+          Hutch::Config.autoload_rails = autoload_rails
+        end
+
+        opts.on('-q', '--quiet', 'Quiet logging') do
+          Hutch::Config.log_level = Logger::WARN
+        end
+
+        opts.on('-v', '--verbose', 'Verbose logging') do
+          Hutch::Config.log_level = Logger::DEBUG
+        end
+
+        opts.on('--namespace NAMESPACE', 'Queue namespace') do |namespace|
+          Hutch::Config.namespace = namespace
+        end
+
+        opts.on('-d', '--daemonise', 'Daemonise') do |daemonise|
+          Hutch::Config.daemonise = daemonise
+        end
+
+        opts.on('--pidfile PIDFILE', 'Pidfile') do |pidfile|
+          Hutch::Config.pidfile = pidfile
+        end
+
+        opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
+          Hutch::Config.group = group
+        end
+
+        opts.on('--version', 'Print the version and exit') do
+          puts "hutch v#{VERSION}"
+          exit 0
+        end
+
+        opts.on('-h', '--help', 'Show this message and exit') do
+          puts opts
+          exit 0
+        end
+      end.parse!(args)
+    end
+
+    def write_pid
+      pidfile = File.expand_path(Hutch::Config.pidfile)
+      Hutch.logger.info "writing pid in #{pidfile}"
+      File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
+    end
+
+    private
+
+    def daemonise_process
+      return unless Hutch::Config.daemonise
+      if defined?(JRUBY_VERSION)
+        Hutch.logger.warn "JRuby ignores the --daemonise option"
+        return
+      end
+
+      ::Process.daemon(true)
+    end
+
+    def abort_without_file(file, file_description, &block)
+      abort_with_message("#{file_description} '#{file}' not found") unless File.exist?(file)
+
+      yield
+    end
+
+    def abort_with_message(message)
+      abort message
+    end
+  end
+end

data/lib/hutch/config.rb

@@ -0,0 +1,304 @@
+require 'hutch/error_handlers/logger'
+require 'hutch/tracers'
+require 'hutch/serializers/json'
+require 'erb'
+require 'logger'
+
+module Hutch
+  class UnknownAttributeError < StandardError; end
+
+  # Configuration settings, available everywhere
+  #
+  # There are defaults, which can be overridden by ENV variables prefixed by
+  # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
+  # method.
+  #
+  # @example Configuring on the command-line
+  #   HUTCH_PUBLISHER_CONFIRMS=false hutch
+  module Config
+    require 'yaml'
+    @string_keys = Set.new
+    @number_keys = Set.new
+    @boolean_keys = Set.new
+    @settings_defaults = {}
+
+    # Define a String user setting
+    # @!visibility private
+    def self.string_setting(name, default_value)
+      @string_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Number user setting
+    # @!visibility private
+    def self.number_setting(name, default_value)
+      @number_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # Define a Boolean user setting
+    # @!visibility private
+    def self.boolean_setting(name, default_value)
+      @boolean_keys << name
+      @settings_defaults[name] = default_value
+    end
+
+    # RabbitMQ hostname
+    string_setting :mq_host, '127.0.0.1'
+
+    # RabbitMQ Exchange to use for publishing
+    string_setting :mq_exchange, 'hutch'
+
+    # RabbitMQ Exchange type to use for publishing
+    string_setting :mq_exchange_type, 'topic'
+
+    # RabbitMQ vhost to use
+    string_setting :mq_vhost, '/'
+
+    # RabbitMQ username to use.
+    #
+    # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
+    string_setting :mq_username, 'guest'
+
+    # RabbitMQ password
+    string_setting :mq_password, 'guest'
+
+    # RabbitMQ Auth Mechanism
+    string_setting :mq_auth_mechanism, 'PLAIN'
+
+    # RabbitMQ URI (takes precedence over MQ username, password, host, port and vhost settings)
+    string_setting :uri, nil
+
+    # RabbitMQ HTTP API hostname
+    string_setting :mq_api_host, '127.0.0.1'
+
+    # RabbitMQ port
+    number_setting :mq_port, 5672
+
+    # RabbitMQ HTTP API port
+    number_setting :mq_api_port, 15672
+
+    # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
+    number_setting :heartbeat, 30
+
+    # The <tt>basic.qos</tt> prefetch value to use.
+    #
+    # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
+    number_setting :channel_prefetch, 0
+
+    # Bunny's socket open timeout
+    number_setting :connection_timeout, 11
+
+    # Bunny's socket read timeout
+    number_setting :read_timeout, 11
+
+    # Bunny's socket write timeout
+    number_setting :write_timeout, 11
+
+    # Bunny's enable/disable network recovery
+    boolean_setting :automatically_recover, true
+
+    # Bunny's reconnect interval
+    number_setting :network_recovery_interval, 1
+
+    # FIXME: DOCUMENT THIS
+    number_setting :graceful_exit_timeout, 11
+
+    # Bunny consumer work pool size
+    number_setting :consumer_pool_size, 1
+
+    # Should TLS be used?
+    boolean_setting :mq_tls, false
+
+    # Should SSL certificate be verified?
+    boolean_setting :mq_verify_peer, true
+
+    # Should SSL be used for the RabbitMQ API?
+    boolean_setting :mq_api_ssl, false
+
+    # Should the current Rails app directory be required?
+    boolean_setting :autoload_rails, true
+
+    # Should the Hutch runner process daemonise?
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :daemonise, false
+
+    # Should RabbitMQ publisher confirms be enabled?
+    #
+    # Leaves it up to the app how they are tracked
+    # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
+    boolean_setting :publisher_confirms, false
+
+    # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
+    # every publish.
+    #
+    # **This is the safest option which also offers the
+    # lowest throughput**.
+    boolean_setting :force_publisher_confirms, false
+
+    # Should the RabbitMQ HTTP API be used?
+    boolean_setting :enable_http_api_use, true
+
+    # Should Bunny's consumer work pool threads abort on exception.
+    #
+    # The option is ignored on JRuby.
+    boolean_setting :consumer_pool_abort_on_exception, false
+
+    # Prefix displayed on the consumers tags.
+    string_setting :consumer_tag_prefix, 'hutch'
+
+    # A namespace to help group your queues
+    string_setting :namespace, nil
+
+    string_setting :group, ''
+
+    # Set of all setting keys
+    ALL_KEYS = @boolean_keys + @number_keys + @string_keys
+
+    def self.initialize(params = {})
+      unless @config
+        @config = default_config
+        define_methods
+        @config.merge!(env_based_config)
+      end
+      @config.merge!(params)
+      @config
+    end
+
+    # Default settings
+    #
+    # @return [Hash]
+    def self.default_config
+      @settings_defaults.merge({
+        mq_exchange_options: {},
+        mq_tls_cert: nil,
+        mq_tls_key: nil,
+        mq_tls_ca_certificates: nil,
+        uri: nil,
+        log_level: Logger::INFO,
+        client_logger: nil,
+        require_paths: [],
+        error_handlers: [Hutch::ErrorHandlers::Logger.new],
+        # note that this is not a list, it is a chain of responsibility
+        # that will fall back to "nack unconditionally"
+        error_acknowledgements: [],
+        setup_procs: [],
+        consumer_groups: {},
+        tracer: Hutch::Tracers::NullTracer,
+        namespace: nil,
+        pidfile: nil,
+        serializer: Hutch::Serializers::JSON
+      })
+    end
+
+    # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
+    #
+    # @return [Hash]
+    def self.env_based_config
+      env_keys_configured.each_with_object({}) {|attr, result|
+        value = ENV[key_for(attr)]
+
+        result[attr] = type_cast(attr, value)
+      }
+    end
+
+    # @return [Array<Symbol>]
+    def self.env_keys_configured
+      ALL_KEYS.each {|attr| check_attr(attr) }
+
+      ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
+    end
+
+    def self.get(attr)
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym]
+    end
+
+    def self.key_for(attr)
+      key = attr.to_s.gsub('.', '__').upcase
+      "HUTCH_#{key}"
+    end
+
+    def self.is_bool(attr)
+      @boolean_keys.include?(attr)
+    end
+
+    def self.to_bool(value)
+      case value
+      when nil, false, '', /^(false|f|no|n|0)$/i
+        false
+      else
+        true
+      end
+    end
+
+    def self.is_num(attr)
+      @number_keys.include?(attr)
+    end
+
+    def self.set(attr, value)
+      check_attr(attr.to_sym)
+      user_config[attr.to_sym] = type_cast(attr, value)
+    end
+
+    class << self
+      alias_method :[],  :get
+      alias_method :[]=, :set
+    end
+
+    def self.check_attr(attr)
+      unless user_config.key?(attr)
+        raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
+      end
+    end
+
+    def self.user_config
+      @config ||= initialize
+    end
+
+    def self.to_hash
+      user_config
+    end
+
+    def self.load_from_file(file)
+      YAML.load(ERB.new(File.read(file)).result).each do |attr, value|
+        Hutch::Config.send("#{attr}=", convert_value(attr, value))
+      end
+    end
+
+    def self.convert_value(attr, value)
+      case attr
+      when 'tracer'
+        Kernel.const_get(value)
+      else
+        value
+      end
+    end
+
+    def self.type_cast(attr, value)
+      case
+      when is_bool(attr) || value == 'false'
+        to_bool(value)
+      when is_num(attr)
+        value.to_i
+      else
+        value
+      end
+    end
+    private_class_method :type_cast
+
+    def self.define_methods
+      @config.keys.each do |key|
+        define_singleton_method(key) do
+          get(key)
+        end
+
+        define_singleton_method("#{key}=") do |val|
+          set(key, val)
+        end
+      end
+    end
+  end
+end
+Hutch::Config.initialize

data/lib/hutch/consumer.rb

@@ -0,0 +1,125 @@
+require 'set'
+
+module Hutch
+  # Include this module in a class to register it as a consumer. Consumers
+  # gain a class method called `consume`, which should be used to register
+  # the routing keys a consumer is interested in.
+  module Consumer
+    attr_accessor :broker, :delivery_info
+
+    def self.included(base)
+      base.extend(ClassMethods)
+      Hutch.register_consumer(base)
+    end
+
+    def reject!
+      @message_rejected = true
+      broker.reject(delivery_info.delivery_tag)
+    end
+
+    def requeue!
+      @message_rejected = true
+      broker.requeue(delivery_info.delivery_tag)
+    end
+
+    def message_rejected?
+      !!@message_rejected
+    end
+
+    def logger
+      Hutch::Logging.logger
+    end
+
+    module ClassMethods
+      # Add one or more routing keys to the set of routing keys the consumer
+      # wants to subscribe to.
+      def consume(*routing_keys)
+        @routing_keys = self.routing_keys.union(routing_keys)
+        # these are opt-in
+        @queue_mode = nil
+        @queue_type = nil
+      end
+
+      attr_reader :queue_mode, :queue_type, :initial_group_size
+
+      # Explicitly set the queue name
+      def queue_name(name)
+        @queue_name = name
+      end
+
+      # Explicitly set the queue mode to 'lazy'
+      def lazy_queue
+        @queue_mode = 'lazy'
+      end
+
+      # Explicitly set the queue type to 'classic'
+      def classic_queue
+        @queue_type = 'classic'
+      end
+
+      # Explicitly set the queue type to 'quorum'
+      # @param [Hash] options the options params related to quorum queue
+      # @option options [Integer] :initial_group_size Initial Replication Factor
+      def quorum_queue(options = {})
+        @queue_type = 'quorum'
+        @initial_group_size = options[:initial_group_size]
+      end
+
+      # Configures an optional argument that will be passed when declaring the queue.
+      # Prefer using a policy to this DSL: https://www.rabbitmq.com/parameters.html#policies
+      def arguments(arguments = {})
+        @arguments = arguments
+      end
+
+      # Congfiures queue options that will be passed when declaring the queue.
+      def queue_options(options = {})
+        @queue_options = options
+      end
+
+      # Set custom serializer class, override global value
+      def serializer(name)
+        @serializer = name
+      end
+
+      # The RabbitMQ queue name for the consumer. This is derived from the
+      # fully-qualified class name. Module separators are replaced with single
+      # colons, camelcased class names are converted to snake case.
+      def get_queue_name
+        return @queue_name unless @queue_name.nil?
+        queue_name = self.name.gsub(/::/, ':')
+        queue_name.gsub!(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }
+        queue_name.downcase
+      end
+
+      # Returns consumer custom arguments.
+      def get_arguments
+        all_arguments = @arguments || {}
+
+        all_arguments['x-queue-mode'] = @queue_mode if @queue_mode
+        all_arguments['x-queue-type'] = @queue_type if @queue_type
+        all_arguments['x-quorum-initial-group-size'] = @initial_group_size if @initial_group_size
+
+        all_arguments
+      end
+
+      def get_options
+        default_options = { durable: true }
+
+        all_options = default_options.merge(@queue_options || {})
+        all_options[:arguments] = get_arguments
+
+        all_options
+      end
+
+      # Accessor for the consumer's routing key.
+      def routing_keys
+        @routing_keys ||= Set.new
+      end
+
+      def get_serializer
+        @serializer
+      end
+    end
+  end
+end
+