dkastner-hutch 0.17.1

data/lib/hutch/broker.rb

@@ -0,0 +1,356 @@
+require 'bunny'
+require 'carrot-top'
+require 'securerandom'
+require 'hutch/logging'
+require 'hutch/exceptions'
+
+module Hutch
+  class Broker
+    include Logging
+
+    attr_accessor :connection, :channel, :exchange, :api_client
+
+    def initialize(config = nil)
+      @config = config || Hutch::Config
+    end
+
+    def connect(options = {})
+      @options = options
+      set_up_amqp_connection
+      if http_api_use_enabled?
+        logger.info "HTTP API use is enabled"
+        set_up_api_connection
+      else
+        logger.info "HTTP API use is disabled"
+      end
+
+      if tracing_enabled?
+        logger.info "tracing is enabled using #{@config[:tracer]}"
+      else
+        logger.info "tracing is disabled"
+      end
+
+      if block_given?
+        begin
+          yield
+        ensure
+          disconnect
+        end
+      end
+    end
+
+    def disconnect
+      @channel.close    if @channel
+      @connection.close if @connection
+      @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
+    end
+
+    # Connect to RabbitMQ via AMQP. This sets up the main connection and
+    # channel we use for talking to RabbitMQ. It also ensures the existance of
+    # the exchange we'll be using.
+    def set_up_amqp_connection
+      open_connection!
+      open_channel!
+
+      exchange_name = @config[:mq_exchange]
+      logger.info "using topic exchange '#{exchange_name}'"
+
+      with_bunny_precondition_handler('exchange') do
+        @exchange = @channel.topic(exchange_name, durable: true)
+      end
+    end
+
+    def open_connection!
+      logger.info "connecting to rabbitmq (#{sanitized_uri})"
+
+      @connection = Bunny.new(connection_params)
+
+      with_bunny_connection_handler(sanitized_uri) do
+        @connection.start
+      end
+
+      logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
+      @connection
+    end
+
+    def open_channel!
+      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
+      @channel = connection.create_channel(nil, consumer_pool_size).tap do |ch|
+        ch.prefetch(@config[:channel_prefetch]) if @config[:channel_prefetch]
+        if @config[:publisher_confirms] || @config[:force_publisher_confirms]
+          logger.info 'enabling publisher confirms'
+          ch.confirm_select
+        end
+      end
+    end
+
+    # Set up the connection to the RabbitMQ management API. Unfortunately, this
+    # is necessary to do a few things that are impossible over AMQP. E.g.
+    # listing queues and bindings.
+    def set_up_api_connection
+      logger.info "connecting to rabbitmq HTTP API (#{api_config.sanitized_uri})"
+
+      with_authentication_error_handler do
+        with_connection_error_handler do
+          @api_client = CarrotTop.new(host: api_config.host, port: api_config.port,
+                                      user: api_config.username, password: api_config.password,
+                                      ssl: api_config.ssl)
+          @api_client.exchanges
+        end
+      end
+    end
+
+    def http_api_use_enabled?
+      op = @options.fetch(:enable_http_api_use, true)
+      cf = if @config[:enable_http_api_use].nil?
+             true
+           else
+             @config[:enable_http_api_use]
+           end
+
+      op && cf
+    end
+
+    def tracing_enabled?
+      @config[:tracer] && @config[:tracer] != Hutch::Tracers::NullTracer
+    end
+
+    # Create / get a durable queue and apply namespace if it exists.
+    def queue(name, arguments = {})
+      with_bunny_precondition_handler('queue') do
+        namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
+        name = name.prepend(namespace + ":") unless namespace.empty?
+        channel.queue(name, durable: true, arguments: arguments)
+      end
+    end
+
+    # Return a mapping of queue names to the routing keys they're bound to.
+    def bindings
+      results = Hash.new { |hash, key| hash[key] = [] }
+      @api_client.bindings.each do |binding|
+        next if binding['destination'] == binding['routing_key']
+        next unless binding['source'] == @config[:mq_exchange]
+        next unless binding['vhost'] == @config[:mq_vhost]
+        results[binding['destination']] << binding['routing_key']
+      end
+      results
+    end
+
+    # Bind a queue to the broker's exchange on the routing keys provided. Any
+    # existing bindings on the queue that aren't present in the array of
+    # routing keys will be unbound.
+    def bind_queue(queue, routing_keys)
+      if http_api_use_enabled?
+        # Find the existing bindings, and unbind any redundant bindings
+        queue_bindings = bindings.select { |dest, keys| dest == queue.name }
+        queue_bindings.each do |dest, keys|
+          keys.reject { |key| routing_keys.include?(key) }.each do |key|
+            logger.debug "removing redundant binding #{queue.name} <--> #{key}"
+            queue.unbind(@exchange, routing_key: key)
+          end
+        end
+      end
+
+      # Ensure all the desired bindings are present
+      routing_keys.each do |routing_key|
+        logger.debug "creating binding #{queue.name} <--> #{routing_key}"
+        queue.bind(@exchange, routing_key: routing_key)
+      end
+    end
+
+    # Each subscriber is run in a thread. This calls Thread#join on each of the
+    # subscriber threads.
+    def wait_on_threads(timeout)
+      # Thread#join returns nil when the timeout is hit. If any return nil,
+      # the threads didn't all join so we return false.
+      per_thread_timeout = timeout.to_f / work_pool_threads.length
+      work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
+    end
+
+    def stop
+      # Enqueue a failing job that kills the consumer loop
+      channel_work_pool.shutdown
+      # Give `timeout` seconds to jobs that are still being processed
+      channel_work_pool.join(@config[:graceful_exit_timeout])
+      # If after `timeout` they are still running, they are killed
+      channel_work_pool.kill
+    end
+
+    def requeue(delivery_tag)
+      @channel.reject(delivery_tag, true)
+    end
+
+    def reject(delivery_tag, requeue=false)
+      @channel.reject(delivery_tag, requeue)
+    end
+
+    def ack(delivery_tag)
+      @channel.ack(delivery_tag, false)
+    end
+
+    def nack(delivery_tag)
+      @channel.nack(delivery_tag, false, false)
+    end
+
+    def publish(routing_key, message, properties = {})
+      ensure_connection!(routing_key, message)
+
+      non_overridable_properties = {
+        routing_key: routing_key,
+        timestamp: Time.now.to_i,
+        content_type: 'application/json'
+      }
+      properties[:message_id] ||= generate_id
+
+      json = JSON.dump(message)
+      logger.info("publishing message '#{json}' to #{routing_key}")
+      response = @exchange.publish(json, {persistent: true}.
+        merge(properties).
+        merge(global_properties).
+        merge(non_overridable_properties))
+
+      channel.wait_for_confirms if @config[:force_publisher_confirms]
+      response
+    end
+
+    def confirm_select(*args)
+      @channel.confirm_select(*args)
+    end
+
+    def wait_for_confirms
+      @channel.wait_for_confirms
+    end
+
+    def using_publisher_confirmations?
+      @channel.using_publisher_confirmations?
+    end
+
+    private
+
+    def raise_publish_error(reason, routing_key, message)
+      msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
+      logger.error(msg)
+      raise PublishError, msg
+    end
+
+    def ensure_connection!(routing_key, message)
+      raise_publish_error('no connection to broker', routing_key, message) unless @connection
+      raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
+    end
+
+    def api_config
+      @api_config ||= OpenStruct.new.tap do |config|
+        config.host = @config[:mq_api_host]
+        config.port = @config[:mq_api_port]
+        config.username = @config[:mq_username]
+        config.password = @config[:mq_password]
+        config.ssl = @config[:mq_api_ssl]
+        config.protocol = config.ssl ? "https://" : "http://"
+        config.sanitized_uri = "#{config.protocol}#{config.username}@#{config.host}:#{config.port}/"
+      end
+    end
+
+    def connection_params
+      parse_uri
+
+      {}.tap do |params|
+        params[:host]               = @config[:mq_host]
+        params[:port]               = @config[:mq_port]
+        params[:vhost]              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
+                                        @config[:mq_vhost]
+                                      else
+                                        Bunny::Session::DEFAULT_VHOST
+                                      end
+        params[:username]           = @config[:mq_username]
+        params[:password]           = @config[:mq_password]
+        params[:tls]                = @config[:mq_tls]
+        params[:tls_key]            = @config[:mq_tls_key]
+        params[:tls_cert]           = @config[:mq_tls_cert]
+        params[:heartbeat]          = @config[:heartbeat]
+        params[:connection_timeout] = @config[:connection_timeout]
+        params[:read_timeout]       = @config[:read_timeout]
+        params[:write_timeout]      = @config[:write_timeout]
+
+
+        params[:automatically_recover] = true
+        params[:network_recovery_interval] = 1
+
+        params[:client_logger] = @config[:client_logger] if @config[:client_logger]
+      end
+    end
+
+    def parse_uri
+      return unless @config[:uri] && !@config[:uri].empty?
+
+      u = URI.parse(@config[:uri])
+
+      @config[:mq_host]     = u.host
+      @config[:mq_port]     = u.port
+      @config[:mq_vhost]    = u.path.sub(/^\//, "")
+      @config[:mq_username] = u.user
+      @config[:mq_password] = u.password
+    end
+
+    def sanitized_uri
+      p = connection_params
+      scheme = p[:tls] ? "amqps" : "amqp"
+
+      "#{scheme}://#{p[:username]}@#{p[:host]}:#{p[:port]}/#{p[:vhost].sub(/^\//, '')}"
+    end
+
+    def with_authentication_error_handler
+      yield
+    rescue Net::HTTPServerException => ex
+      logger.error "HTTP API connection error: #{ex.message.downcase}"
+      if ex.response.code == '401'
+        raise AuthenticationError.new('invalid HTTP API credentials')
+      else
+        raise
+      end
+    end
+
+    def with_connection_error_handler
+      yield
+    rescue Errno::ECONNREFUSED => ex
+      logger.error "HTTP API connection error: #{ex.message.downcase}"
+      raise ConnectionError.new("couldn't connect to HTTP API at #{api_config.sanitized_uri}")
+    end
+
+    def with_bunny_precondition_handler(item)
+      yield
+    rescue Bunny::PreconditionFailed => ex
+      logger.error ex.message
+      s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
+          "Perhaps it is being redeclared with non-matching attributes"
+      raise WorkerSetupError.new(s)
+    end
+
+    def with_bunny_connection_handler(uri)
+      yield
+    rescue Bunny::TCPConnectionFailed => ex
+      logger.error "amqp connection error: #{ex.message.downcase}"
+      raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
+    end
+
+    def work_pool_threads
+      channel_work_pool.threads || []
+    end
+
+    def channel_work_pool
+      @channel.work_pool
+    end
+
+    def consumer_pool_size
+      @config[:consumer_pool_size]
+    end
+
+    def generate_id
+      SecureRandom.uuid
+    end
+
+    def global_properties
+      Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
+    end
+  end
+end
+

data/lib/hutch/cli.rb

@@ -0,0 +1,205 @@
+require 'optparse'
+
+require 'hutch/version'
+require 'hutch/logging'
+require 'hutch/exceptions'
+require 'hutch/config'
+
+module Hutch
+  class CLI
+    include Logging
+
+    # Run a Hutch worker with the command line interface.
+    def run(argv = ARGV)
+      parse_options(argv)
+
+      ::Process.daemon(true) if Hutch::Config.daemonise
+
+      write_pid if Hutch::Config.pidfile
+
+      Hutch.logger.info "hutch booted with pid #{::Process.pid}"
+
+      if load_app && start_work_loop == :success
+        # If we got here, the worker was shut down nicely
+        Hutch.logger.info 'hutch shut down gracefully'
+        exit 0
+      else
+        Hutch.logger.info 'hutch terminated due to an error'
+        exit 1
+      end
+    end
+
+    def load_app
+      # Try to load a Rails app in the current directory
+      load_rails_app('.') if Hutch::Config.autoload_rails
+      Hutch::Config.require_paths.each do |path|
+        # See if each path is a Rails app. If so, try to load it.
+        next if load_rails_app(path)
+
+        # Given path is not a Rails app, try requiring it as a file
+        logger.info "requiring '#{path}'"
+        begin
+          # Need to add '.' to load path for relative requires
+          $LOAD_PATH << '.'
+          require path
+        rescue LoadError
+          logger.fatal "could not load file '#{path}'"
+          return false
+        ensure
+          # Clean up load path
+          $LOAD_PATH.pop
+        end
+      end
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def load_rails_app(path)
+      # path should point to the app's top level directory
+      if File.directory?(path)
+        # Smells like a Rails app if it's got a config/environment.rb file
+        rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
+        if File.exists?(rails_path)
+          logger.info "found rails project (#{path}), booting app"
+          ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          require rails_path
+          ::Rails.application.eager_load!
+          return true
+        end
+      end
+      false
+    end
+
+    # Kick off the work loop. This method returns when the worker is shut down
+    # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
+    def start_work_loop
+      Hutch.connect
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker.run
+      :success
+    rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      logger.fatal ex.message
+      :error
+    end
+
+    def parse_options(args = ARGV)
+      OptionParser.new do |opts|
+        opts.banner = 'usage: hutch [options]'
+
+        opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
+          Hutch::Config.mq_host = host
+        end
+
+        opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
+          Hutch::Config.mq_port = port
+        end
+
+        opts.on("-t", "--[no-]mq-tls", 'Use TLS for the AMQP connection') do |tls|
+          Hutch::Config.mq_tls = tls
+        end
+
+        opts.on('--mq-tls-cert FILE', 'Certificate  for TLS client verification') do |file|
+          abort "Certificate file '#{file}' not found" unless File.exists?(file)
+          Hutch::Config.mq_tls_cert = file
+        end
+
+        opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
+          abort "Private key file '#{file}' not found" unless File.exists?(file)
+          Hutch::Config.mq_tls_key = file
+        end
+
+        opts.on('--mq-exchange EXCHANGE',
+                'Set the RabbitMQ exchange') do |exchange|
+          Hutch::Config.mq_exchange = exchange
+        end
+
+        opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
+          Hutch::Config.mq_vhost = vhost
+        end
+
+        opts.on('--mq-username USERNAME',
+                'Set the RabbitMQ username') do |username|
+          Hutch::Config.mq_username = username
+        end
+
+        opts.on('--mq-password PASSWORD',
+                'Set the RabbitMQ password') do |password|
+          Hutch::Config.mq_password = password
+        end
+
+        opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
+          Hutch::Config.mq_api_host = host
+        end
+
+        opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
+          Hutch::Config.mq_api_port = port
+        end
+
+        opts.on("-s", "--[no-]mq-api-ssl", 'Use SSL for the RabbitMQ API') do |api_ssl|
+          Hutch::Config.mq_api_ssl = api_ssl
+        end
+
+        opts.on('--config FILE', 'Load Hutch configuration from a file') do |file|
+          begin
+            File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
+          rescue Errno::ENOENT
+            abort "Config file '#{file}' not found"
+          end
+        end
+
+        opts.on('--require PATH', 'Require a Rails app or path') do |path|
+          Hutch::Config.require_paths << path
+        end
+
+        opts.on('--[no-]autoload-rails', 'Require the current rails app directory') do |autoload_rails|
+          Hutch::Config.autoload_rails = autoload_rails
+        end
+
+        opts.on('-q', '--quiet', 'Quiet logging') do
+          Hutch::Config.log_level = Logger::WARN
+        end
+
+        opts.on('-v', '--verbose', 'Verbose logging') do
+          Hutch::Config.log_level = Logger::DEBUG
+        end
+
+        opts.on('--namespace NAMESPACE', 'Queue namespace') do |namespace|
+          Hutch::Config.namespace = namespace
+        end
+
+        opts.on('-d', '--daemonise', 'Daemonise') do |daemonise|
+          Hutch::Config.daemonise = daemonise
+        end
+
+        opts.on('--pidfile PIDFILE', 'Pidfile') do |pidfile|
+          Hutch::Config.pidfile = pidfile
+        end
+
+        opts.on('--version', 'Print the version and exit') do
+          puts "hutch v#{VERSION}"
+          exit 0
+        end
+
+        opts.on('-h', '--help', 'Show this message and exit') do
+          puts opts
+          exit 0
+        end
+      end.parse!(args)
+    end
+
+    def write_pid
+      pidfile = File.expand_path(Hutch::Config.pidfile)
+      Hutch.logger.info "writing pid in #{pidfile}"
+      File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
+    end
+
+  end
+end

data/lib/hutch/config.rb

@@ -0,0 +1,121 @@
+require 'hutch/error_handlers/logger'
+require 'erb'
+require 'logger'
+
+module Hutch
+  class UnknownAttributeError < StandardError; end
+
+  module Config
+    require 'yaml'
+
+    def self.initialize(params={})
+      @config = {
+        mq_host: 'localhost',
+        mq_port: 5672,
+        mq_exchange: 'hutch',  # TODO: should this be required?
+        mq_vhost: '/',
+        mq_tls: false,
+        mq_tls_cert: nil,
+        mq_tls_key: nil,
+        mq_username: 'guest',
+        mq_password: 'guest',
+        mq_api_host: 'localhost',
+        mq_api_port: 15672,
+        mq_api_ssl: false,
+        heartbeat: 30,
+        # placeholder, allows specifying connection parameters
+        # as a URI.
+        uri: nil,
+        log_level: Logger::INFO,
+        require_paths: [],
+        autoload_rails: true,
+        error_handlers: [Hutch::ErrorHandlers::Logger.new],
+        tracer: Hutch::Tracers::NullTracer,
+        namespace: nil,
+        daemonise: false,
+        pidfile: nil,
+        channel_prefetch: 0,
+        # enables publisher confirms, leaves it up to the app
+        # how they are tracked
+        publisher_confirms: false,
+        # like `publisher_confirms` above but also
+        # forces waiting for a confirm for every publish
+        force_publisher_confirms: false,
+        # Heroku needs > 10. MK.
+        connection_timeout: 11,
+        read_timeout: 11,
+        write_timeout: 11,
+        enable_http_api_use: true,
+        # Number of seconds that a running consumer is given
+        # to finish its job when gracefully exiting Hutch, before
+        # it's killed.
+        graceful_exit_timeout: 11,
+        client_logger: nil,
+
+        consumer_pool_size: 1,
+      }.merge(params)
+    end
+
+    def self.get(attr)
+      check_attr(attr)
+      user_config[attr]
+    end
+
+    def self.set(attr, value)
+      check_attr(attr)
+      user_config[attr] = value
+    end
+
+    class << self
+      alias_method :[],  :get
+      alias_method :[]=, :set
+    end
+
+    def self.check_attr(attr)
+      unless user_config.key?(attr)
+        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+      end
+    end
+
+    def self.user_config
+      initialize unless @config
+      @config
+    end
+
+    def self.to_hash
+      self.user_config
+    end
+
+    def self.load_from_file(file)
+      YAML.load(ERB.new(File.read(file)).result).each do |attr, value|
+        Hutch::Config.send("#{attr}=", convert_value(attr, value))
+      end
+    end
+
+    def self.convert_value(attr, value)
+      case attr
+      when "tracer"
+        Kernel.const_get(value)
+      else
+        value
+      end
+    end
+
+    def self.method_missing(method, *args, &block)
+      attr = method.to_s.sub(/=$/, '').to_sym
+      return super unless user_config.key?(attr)
+
+      if method =~ /=$/
+        set(attr, args.first)
+      else
+        get(attr)
+      end
+    end
+
+    private
+
+    def deep_copy(obj)
+      Marshal.load(Marshal.dump(obj))
+    end
+  end
+end