hutch 0.1.0

data/lib/hutch/cli.rb

@@ -0,0 +1,151 @@
+require 'optparse'
+
+require 'hutch/logging'
+require 'hutch/exceptions'
+
+module Hutch
+  class CLI
+    include Logging
+
+    # Run a Hutch worker with the command line interface.
+    def run
+      parse_options
+
+      Hutch.logger.info "hutch booted with pid #{Process.pid}"
+
+      if load_app && start_work_loop == :success
+        # If we got here, the worker was shut down nicely
+        Hutch.logger.info 'hutch shut down gracefully'
+        exit 0
+      else
+        Hutch.logger.info 'hutch terminated due to an error'
+        exit 1
+      end
+    end
+
+    def load_app
+      # Try to load a Rails app in the current directory
+      load_rails_app('.')
+      Hutch::Config.require_paths.each do |path|
+        # See if each path is a Rails app. If so, try to load it.
+        next if load_rails_app(path)
+
+        # Given path is not a Rails app, try requiring it as a file
+        logger.info "requiring '#{path}'"
+        begin
+          # Need to add '.' to load path for relative requires
+          $LOAD_PATH << '.'
+          require path
+        rescue LoadError
+          logger.fatal "could not load file '#{path}'"
+          return false
+        ensure
+          # Clean up load path
+          $LOAD_PATH.pop
+        end
+      end
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def load_rails_app(path)
+      # path should point to the app's top level directory
+      if File.directory?(path)
+        # Smells like a Rails app if it's got a config/environment.rb file
+        rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
+        if File.exists?(rails_path)
+          logger.info "found rails project (#{path}), booting app"
+          ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          require rails_path
+          ::Rails.application.eager_load!
+          return true
+        end
+      end
+      false
+    end
+
+    # Kick off the work loop. This method returns when the worker is shut down
+    # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
+    def start_work_loop
+      Hutch.connect
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker.run
+      :success
+    rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      logger.fatal ex.message
+      :error
+    end
+
+    def parse_options
+      OptionParser.new do |opts|
+        opts.banner = 'usage: hutch [options]'
+
+        opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
+          Hutch::Config.mq_host = host
+        end
+
+        opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
+          Hutch::Config.mq_port = port
+        end
+
+        opts.on('--mq-exchange EXCHANGE',
+                'Set the RabbitMQ exchange') do |exchange|
+          Hutch::Config.mq_exchange = exchange
+        end
+
+        opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
+          Hutch::Config.mq_vhost = vhost
+        end
+
+        opts.on('--mq-username USERNAME',
+                'Set the RabbitMQ username') do |username|
+          Hutch::Config.mq_username = username
+        end
+
+        opts.on('--mq-password PASSWORD',
+                'Set the RabbitMQ password') do |password|
+          Hutch::Config.mq_password = password
+        end
+
+        opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
+          Hutch::Config.mq_api_host = host
+        end
+
+        opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
+          Hutch::Config.mq_api_port = port
+        end
+
+        opts.on('--require PATH', 'Require a Rails app or path') do |path|
+          Hutch::Config.require_paths << path
+        end
+
+        opts.on('-q', '--quiet', 'Quiet logging') do
+          Hutch::Config.log_level = Logger::WARN
+        end
+
+        opts.on('-v', '--verbose', 'Verbose logging') do
+          Hutch::Config.log_level = Logger::DEBUG
+        end
+
+        opts.on('--version', 'Print the version and exit') do
+          puts "hutch v#{VERSION}"
+          exit 0
+        end
+
+        opts.on('-h', '--help', 'Show this message and exit') do
+          puts opts
+          exit 0
+        end
+      end.parse!
+    end
+  end
+end
+

data/lib/hutch/config.rb

@@ -0,0 +1,66 @@
+require 'logger'
+
+module Hutch
+  class UnknownAttributeError < StandardError; end
+
+  module Config
+    def self.initialize
+      @config = {
+        mq_host: 'localhost',
+        mq_port: 5672,
+        mq_exchange: 'hutch',  # TODO: should this be required?
+        mq_vhost: '/',
+        mq_username: 'guest',
+        mq_password: 'guest',
+        mq_api_host: 'localhost',
+        mq_api_port: 15672,
+        log_level: Logger::INFO,
+        require_paths: [],
+        error_handlers: [Hutch::ErrorHandlers::Logger.new]
+      }
+    end
+
+    def self.get(attr)
+      check_attr(attr)
+      user_config[attr]
+    end
+
+    def self.set(attr, value)
+      check_attr(attr)
+      user_config[attr] = value
+    end
+
+    class << self
+      alias_method :[],  :get
+      alias_method :[]=, :set
+    end
+
+    def self.check_attr(attr)
+      unless user_config.key?(attr)
+        raise UnknownAttributeError, "#{attr} is not a valid config attribute"
+      end
+    end
+
+    def self.user_config
+      initialize unless @config
+      @config
+    end
+
+    def self.method_missing(method, *args, &block)
+      attr = method.to_s.sub(/=$/, '').to_sym
+      return super unless user_config.key?(attr)
+
+      if method =~ /=$/
+        set(attr, args.first)
+      else
+        get(attr)
+      end
+    end
+
+    private
+
+    def deep_copy(obj)
+      Marshal.load(Marshal.dump(obj))
+    end
+  end
+end

data/lib/hutch/consumer.rb

@@ -0,0 +1,33 @@
+module Hutch
+  # Include this module in a class to register it as a consumer. Consumers
+  # gain a class method called `consume`, which should be used to register
+  # the routing keys a consumer is interested in.
+  module Consumer
+    def self.included(base)
+      base.extend(ClassMethods)
+      Hutch.register_consumer(base)
+    end
+
+    module ClassMethods
+      # Add one or more routing keys to the set of routing keys the consumer
+      # wants to subscribe to.
+      def consume(*routing_keys)
+        @routing_keys = self.routing_keys.union(routing_keys)
+      end
+
+      # The RabbitMQ queue name for the consumer. This is derived from the
+      # fully-qualified class name. Module separators are replaced with single
+      # colons, camelcased class names are converted to snake case.
+      def queue_name
+        queue_name = self.name.gsub(/::/, ':')
+        queue_name.gsub(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }.downcase
+      end
+
+      # Accessor for the consumer's routing key.
+      def routing_keys
+        @routing_keys ||= Set.new
+      end
+    end
+  end
+end
+

data/lib/hutch/error_handlers/logger.rb

@@ -0,0 +1,16 @@
+require 'hutch/logging'
+
+module Hutch
+  module ErrorHandlers
+    class Logger
+      include Logging
+
+      def handle(message_id, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "error in consumer '#{consumer}'"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        logger.error (['backtrace:'] + ex.backtrace).join("\n")
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/sentry.rb

@@ -0,0 +1,23 @@
+require 'hutch/logging'
+require 'raven'
+
+module Hutch
+  module ErrorHandlers
+    class Sentry
+      include Logging
+
+      def initialize
+        unless Raven.respond_to?(:capture_exception)
+          raise "The Hutch Sentry error handler requires Raven >= 0.4.0"
+        end
+      end
+
+      def handle(message_id, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Sentry"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        Raven.capture_exception(ex)
+      end
+    end
+  end
+end

data/lib/hutch/exceptions.rb

@@ -0,0 +1,5 @@
+module Hutch
+  class ConnectionError < StandardError; end
+  class AuthenticationError < StandardError; end
+  class WorkerSetupError < StandardError; end
+end

data/lib/hutch/logging.rb

@@ -0,0 +1,32 @@
+require 'logger'
+require 'time'
+require 'hutch/config'
+
+module Hutch
+  module Logging
+    class HutchFormatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} #{severity} -- #{message}\n"
+      end
+    end
+
+    def self.setup_logger(target = $stdout)
+      @logger = Logger.new(target)
+      @logger.level = Hutch::Config.log_level
+      @logger.formatter = HutchFormatter.new
+      @logger
+    end
+
+    def self.logger
+      @logger || setup_logger
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def logger
+      Hutch::Logging.logger
+    end
+  end
+end

data/lib/hutch/message.rb

@@ -0,0 +1,26 @@
+require 'multi_json'
+require 'forwardable'
+
+module Hutch
+  class Message
+    extend Forwardable
+
+    def initialize(delivery_info, properties, payload)
+      @delivery_info = delivery_info
+      @properties    = properties
+      @payload       = payload
+      @body          = MultiJson.load(payload, symbolize_keys: true)
+    end
+
+    def_delegator :@body, :[]
+    def_delegators :@properties, :message_id, :timestamp
+    def_delegators :@delivery_info, :routing_key, :exchange
+
+    attr_reader :body
+
+    def to_s
+      "#<Message #{body.map { |k,v| "#{k}: #{v.inspect}" }.join(', ')}>"
+    end
+  end
+end
+

data/lib/hutch/version.rb

@@ -0,0 +1,4 @@
+module Hutch
+  VERSION = '0.1.0'.freeze
+end
+

data/lib/hutch/worker.rb

@@ -0,0 +1,104 @@
+require 'hutch/message'
+require 'hutch/logging'
+require 'hutch/broker'
+require 'carrot-top'
+
+module Hutch
+  class Worker
+    include Logging
+
+    def initialize(broker, consumers)
+      @broker = broker
+      raise WorkerSetupError.new('no consumers loaded') if consumers.empty?
+      @consumers = consumers
+    end
+
+    # Run the main event loop. The consumers will be set up with queues, and
+    # process the messages in their respective queues indefinitely. This method
+    # never returns.
+    def run
+      setup_queues
+
+      # Set up signal handlers for graceful shutdown
+      register_signal_handlers
+
+      # Take a break from Thread#join every 0.1 seconds to check if we've
+      # been sent any signals
+      handle_signals until @broker.wait_on_threads(0.1)
+    rescue Bunny::PreconditionFailed => ex
+      logger.error ex.message
+      raise WorkerSetupError.new('could not create queue due to a type ' +
+                                 'conflict with an existing queue, remove ' +
+                                 'the existing queue and try again')
+    end
+
+    # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
+    # gracefully. Forceful shutdowns are very bad!
+    def register_signal_handlers
+      Thread.main[:signal_queue] = []
+      %w(QUIT TERM INT).map(&:to_sym).each do |sig|
+        # This needs to be reentrant, so we queue up signals to be handled
+        # in the run loop, rather than acting on signals here
+        trap(sig) do
+          Thread.main[:signal_queue] << sig
+        end
+      end
+    end
+
+    # Handle any pending signals
+    def handle_signals
+      signal = Thread.main[:signal_queue].shift
+      if signal
+        logger.info "caught sig#{signal.downcase}, stopping hutch..."
+        stop
+      end
+    end
+
+    # Stop a running worker by killing all subscriber threads.
+    def stop
+      @broker.stop
+    end
+
+    # Set up the queues for each of the worker's consumers.
+    def setup_queues
+      logger.info 'setting up queues'
+      @consumers.each { |consumer| setup_queue(consumer) }
+    end
+
+    # Bind a consumer's routing keys to its queue, and set up a subscription to
+    # receive messages sent to the queue.
+    def setup_queue(consumer)
+      queue = @broker.queue(consumer.queue_name)
+      @broker.bind_queue(queue, consumer.routing_keys)
+
+      queue.subscribe(ack: true) do |delivery_info, properties, payload|
+        handle_message(consumer, delivery_info, properties, payload)
+      end
+    end
+
+    # Called internally when a new messages comes in from RabbitMQ. Responsible
+    # for wrapping up the message and passing it to the consumer.
+    def handle_message(consumer, delivery_info, properties, payload)
+      logger.info("message(#{properties.message_id || '-'}): " +
+                  "routing key: #{delivery_info.routing_key}, " +
+                  "consumer: #{consumer}, " +
+                  "payload: #{payload}")
+
+      message = Message.new(delivery_info, properties, payload)
+      broker = @broker
+      begin
+        consumer.new.process(message)
+        broker.ack(delivery_info.delivery_tag)
+      rescue StandardError => ex
+        handle_error(message.message_id, consumer, ex)
+        broker.ack(delivery_info.delivery_tag)
+      end
+    end
+
+    def handle_error(message_id, consumer, ex)
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle(message_id, consumer, ex)
+      end
+    end
+  end
+end