dkastner-hutch 0.17.1

data/lib/hutch/consumer.rb

@@ -0,0 +1,66 @@
+require 'set'
+
+module Hutch
+  # Include this module in a class to register it as a consumer. Consumers
+  # gain a class method called `consume`, which should be used to register
+  # the routing keys a consumer is interested in.
+  module Consumer
+    attr_accessor :broker, :delivery_info
+
+    def self.included(base)
+      base.extend(ClassMethods)
+      Hutch.register_consumer(base)
+    end
+
+    def reject!
+      broker.reject(delivery_info.delivery_tag)
+    end
+
+    def requeue!
+      broker.requeue(delivery_info.delivery_tag)
+    end
+    
+    def logger
+      Hutch::Logging.logger
+    end
+
+    module ClassMethods
+      # Add one or more routing keys to the set of routing keys the consumer
+      # wants to subscribe to.
+      def consume(*routing_keys)
+        @routing_keys = self.routing_keys.union(routing_keys)
+      end
+
+      # Explicitly set the queue name
+      def queue_name(name)
+        @queue_name = name
+      end
+
+      # Allow to specify custom arguments that will be passed when creating the queue.
+      def arguments(arguments = {})
+        @arguments = arguments
+      end
+
+      # The RabbitMQ queue name for the consumer. This is derived from the
+      # fully-qualified class name. Module separators are replaced with single
+      # colons, camelcased class names are converted to snake case.
+      def get_queue_name
+        return @queue_name unless @queue_name.nil?
+        queue_name = self.name.gsub(/::/, ':')
+        queue_name.gsub!(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }
+        queue_name.downcase
+      end
+
+      # Returns consumer custom arguments.
+      def get_arguments
+        @arguments || {}
+      end
+
+      # Accessor for the consumer's routing key.
+      def routing_keys
+        @routing_keys ||= Set.new
+      end
+    end
+  end
+end
+

data/lib/hutch/error_handlers/airbrake.rb

@@ -0,0 +1,26 @@
+require 'hutch/logging'
+require 'airbrake'
+
+module Hutch
+  module ErrorHandlers
+    class Airbrake
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Airbrake"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        ::Airbrake.notify_or_ignore(ex, {
+          :error_class => ex.class.name,
+          :error_message => "#{ ex.class.name }: #{ ex.message }",
+          :backtrace => ex.backtrace,
+          :parameters => {
+            :payload => payload,
+            :consumer => consumer,
+          },
+          :cgi_data => ENV.to_hash,
+        })
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/honeybadger.rb

@@ -0,0 +1,28 @@
+require 'hutch/logging'
+require 'honeybadger'
+
+module Hutch
+  module ErrorHandlers
+    class Honeybadger
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Honeybadger"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        ::Honeybadger.notify_or_ignore(
+          :error_class => ex.class.name,
+          :error_message => "#{ ex.class.name }: #{ ex.message }",
+          :backtrace => ex.backtrace,
+          :context => {
+            :message_id => message_id,
+            :consumer => consumer
+          },
+          :parameters => {
+            :payload => payload
+          }
+        )
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/logger.rb

@@ -0,0 +1,16 @@
+require 'hutch/logging'
+
+module Hutch
+  module ErrorHandlers
+    class Logger
+      include Logging
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "error in consumer '#{consumer}'"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        logger.error (['backtrace:'] + ex.backtrace).join("\n")
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/sentry.rb

@@ -0,0 +1,23 @@
+require 'hutch/logging'
+require 'raven'
+
+module Hutch
+  module ErrorHandlers
+    class Sentry
+      include Logging
+
+      def initialize
+        unless Raven.respond_to?(:capture_exception)
+          raise "The Hutch Sentry error handler requires Raven >= 0.4.0"
+        end
+      end
+
+      def handle(message_id, payload, consumer, ex)
+        prefix = "message(#{message_id || '-'}): "
+        logger.error prefix + "Logging event to Sentry"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        Raven.capture_exception(ex)
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers.rb

@@ -0,0 +1,8 @@
+module Hutch
+  module ErrorHandlers
+    autoload :Logger,      'hutch/error_handlers/logger'
+    autoload :Sentry,      'hutch/error_handlers/sentry'
+    autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
+    autoload :Airbrake,    'hutch/error_handlers/airbrake'
+  end
+end

data/lib/hutch/exceptions.rb

@@ -0,0 +1,7 @@
+module Hutch
+  class Exception < StandardError; end
+  class ConnectionError < Exception; end
+  class AuthenticationError < Exception; end
+  class WorkerSetupError < Exception; end
+  class PublishError < Exception; end
+end

data/lib/hutch/logging.rb

@@ -0,0 +1,32 @@
+require 'logger'
+require 'time'
+
+module Hutch
+  module Logging
+    class HutchFormatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} #{severity} -- #{message}\n"
+      end
+    end
+
+    def self.setup_logger(target = $stdout)
+      require 'hutch/config'
+      @logger = Logger.new(target)
+      @logger.level = Hutch::Config.log_level
+      @logger.formatter = HutchFormatter.new
+      @logger
+    end
+
+    def self.logger
+      @logger || setup_logger
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def logger
+      Hutch::Logging.logger
+    end
+  end
+end

data/lib/hutch/message.rb

@@ -0,0 +1,33 @@
+require 'multi_json'
+require 'forwardable'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Hutch
+  class Message
+    extend Forwardable
+
+    attr_reader :delivery_info, :properties, :payload
+
+    def initialize(delivery_info, properties, payload)
+      @delivery_info = delivery_info
+      @properties    = properties
+      @payload       = payload
+      @body          = MultiJson.load(payload).with_indifferent_access
+    end
+
+    def_delegator :@body, :[]
+    def_delegators :@properties, :message_id, :timestamp
+    def_delegators :@delivery_info, :routing_key, :exchange
+
+    attr_reader :body
+
+    def to_s
+      attrs = { :@body => body.to_s, message_id: message_id,
+                timestamp: timestamp, routing_key: routing_key }
+      "#<Message #{attrs.map { |k,v| "#{k}=#{v.inspect}" }.join(', ')}>"
+    end
+
+    alias_method :inspect, :to_s
+  end
+end
+

data/lib/hutch/tracers/newrelic.rb

@@ -0,0 +1,19 @@
+require 'newrelic_rpm'
+
+module Hutch
+  module Tracers
+    class NewRelic
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        @klass.process(message)
+      end
+
+      add_transaction_tracer :handle, :category => 'OtherTransaction/HutchConsumer', :path => '#{@klass.class.name}'
+    end
+  end
+end

data/lib/hutch/tracers/null_tracer.rb

@@ -0,0 +1,15 @@
+module Hutch
+  module Tracers
+    class NullTracer
+
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        @klass.process(message)
+      end
+
+    end
+  end
+end

data/lib/hutch/tracers.rb

@@ -0,0 +1,6 @@
+module Hutch
+  module Tracers
+    autoload :NullTracer, 'hutch/tracers/null_tracer'
+    autoload :NewRelic,   'hutch/tracers/newrelic'
+  end
+end

data/lib/hutch/version.rb

@@ -0,0 +1,4 @@
+module Hutch
+  VERSION = '0.17.1'.freeze
+end
+

data/lib/hutch/worker.rb

@@ -0,0 +1,110 @@
+require 'hutch/message'
+require 'hutch/logging'
+require 'hutch/broker'
+require 'carrot-top'
+
+module Hutch
+  class Worker
+    include Logging
+
+    def initialize(broker, consumers)
+      @broker        = broker
+      self.consumers = consumers
+    end
+
+    # Run the main event loop. The consumers will be set up with queues, and
+    # process the messages in their respective queues indefinitely. This method
+    # never returns.
+    def run
+      setup_queues
+
+      # Set up signal handlers for graceful shutdown
+      register_signal_handlers
+
+      # Take a break from Thread#join every 0.1 seconds to check if we've
+      # been sent any signals
+      handle_signals until @broker.wait_on_threads(0.1)
+    end
+
+    # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
+    # gracefully. Forceful shutdowns are very bad!
+    def register_signal_handlers
+      Thread.main[:signal_queue] = []
+      %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.map(&:to_sym).each do |sig|
+        # This needs to be reentrant, so we queue up signals to be handled
+        # in the run loop, rather than acting on signals here
+        trap(sig) do
+          Thread.main[:signal_queue] << sig
+        end
+      end
+    end
+
+    # Handle any pending signals
+    def handle_signals
+      signal = Thread.main[:signal_queue].shift
+      if signal
+        logger.info "caught sig#{signal.downcase}, stopping hutch..."
+        stop
+      end
+    end
+
+    # Stop a running worker by killing all subscriber threads.
+    def stop
+      @broker.stop
+    end
+
+    # Set up the queues for each of the worker's consumers.
+    def setup_queues
+      logger.info 'setting up queues'
+      @consumers.each { |consumer| setup_queue(consumer) }
+    end
+
+    # Bind a consumer's routing keys to its queue, and set up a subscription to
+    # receive messages sent to the queue.
+    def setup_queue(consumer)
+      queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
+      @broker.bind_queue(queue, consumer.routing_keys)
+
+      queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
+        handle_message(consumer, delivery_info, properties, payload)
+      end
+    end
+
+    # Called internally when a new messages comes in from RabbitMQ. Responsible
+    # for wrapping up the message and passing it to the consumer.
+    def handle_message(consumer, delivery_info, properties, payload)
+      logger.info("message(#{properties.message_id || '-'}): " +
+                  "routing key: #{delivery_info.routing_key}, " +
+                  "consumer: #{consumer}, " +
+                  "payload: #{payload}")
+
+      broker = @broker
+      begin
+        message = Message.new(delivery_info, properties, payload)
+        consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
+        with_tracing(consumer_instance).handle(message)
+        broker.ack(delivery_info.delivery_tag)
+      rescue StandardError => ex
+        broker.nack(delivery_info.delivery_tag)
+        handle_error(properties.message_id, payload, consumer, ex)
+      end
+    end
+
+    def with_tracing(klass)
+      Hutch::Config[:tracer].new(klass)
+    end
+
+    def handle_error(message_id, payload, consumer, ex)
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle(message_id, payload, consumer, ex)
+      end
+    end
+
+    def consumers=(val)
+      if val.empty?
+        logger.warn "no consumer loaded, ensure there's no configuration issue"
+      end
+      @consumers = val
+    end
+  end
+end

data/lib/hutch.rb

@@ -0,0 +1,60 @@
+require 'hutch/consumer'
+require 'hutch/worker'
+require 'hutch/broker'
+require 'hutch/logging'
+require 'hutch/config'
+require 'hutch/message'
+require 'hutch/cli'
+require 'hutch/version'
+require 'hutch/error_handlers'
+require 'hutch/exceptions'
+require 'hutch/tracers'
+
+module Hutch
+
+  def self.register_consumer(consumer)
+    self.consumers << consumer
+  end
+
+  def self.consumers
+    @consumers ||= []
+  end
+
+  def self.logger
+    Hutch::Logging.logger
+  end
+
+  def self.global_properties=(properties)
+    @global_properties = properties
+  end
+
+  def self.global_properties
+    @global_properties ||= {}
+  end
+
+  def self.connect(options = {}, config = Hutch::Config)
+    unless connected?
+      @broker = Hutch::Broker.new(config)
+      @broker.connect(options)
+    end
+  end
+
+  def self.disconnect
+    @broker.disconnect if @broker
+  end
+
+  def self.broker
+    @broker
+  end
+
+  def self.connected?
+    return false unless broker
+    return false unless broker.connection
+    broker.connection.open?
+  end
+
+  def self.publish(*args)
+    broker.publish(*args)
+  end
+end
+