sneakers_custom_bunny 1.0.4

data/lib/sneakers/handlers/maxretry.rb

@@ -0,0 +1,191 @@
+require 'base64'
+require 'json'
+
+module Sneakers
+  module Handlers
+    #
+    # Maxretry uses dead letter policies on Rabbitmq to requeue and retry
+    # messages after failure (rejections, errors and timeouts). When the maximum
+    # number of retries is reached it will put the message on an error queue.
+    # This handler will only retry at the queue level. To accomplish that, the
+    # setup is a bit complex.
+    #
+    # Input:
+    #   worker_exchange (eXchange)
+    #   worker_queue (Queue)
+    # We create:
+    #   worker_queue-retry - (X) where we setup the worker queue to dead-letter.
+    #   worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to
+    #                        worker_queue-retry-requeue.
+    #   worker_queue-error - (X) where to send max-retry failures
+    #   worker_queue-error - (Q) bound to worker_queue-error.
+    #   worker_queue-retry-requeue - (X) exchange to bind worker_queue to for
+    #                                requeuing directly to the worker_queue.
+    #
+    # This requires that you setup arguments to the worker queue to line up the
+    # dead letter queue. See the example for more information.
+    #
+    # Many of these can be override with options:
+    # - retry_exchange - sets retry exchange & queue
+    # - retry_error_exchange - sets error exchange and queue
+    # - retry_requeue_exchange - sets the exchange created to re-queue things
+    #   back to the worker queue.
+    #
+    class Maxretry
+
+      def initialize(channel, queue, opts)
+        @worker_queue_name = queue.name
+        Sneakers.logger.debug do
+          "#{log_prefix} creating handler, opts=#{opts}"
+        end
+
+        @channel = channel
+        @opts = opts
+
+        # Construct names, defaulting where suitable
+        retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry"
+        error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error"
+        requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue"
+
+        # Create the exchanges
+        @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name|
+          Sneakers.logger.debug { "#{log_prefix} creating exchange=#{name}" }
+          @channel.exchange(name,
+                            :type => 'topic',
+                            :durable => opts[:durable])
+        end
+
+        # Create the queues and bindings
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}"
+        end
+        @retry_queue = @channel.queue(retry_name,
+                                     :durable => opts[:durable],
+                                     :arguments => {
+                                       :'x-dead-letter-exchange' => requeue_name,
+                                       :'x-message-ttl' => @opts[:retry_timeout] || 60000
+                                     })
+        @retry_queue.bind(@retry_exchange, :routing_key => '#')
+
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{error_name}"
+        end
+        @error_queue = @channel.queue(error_name,
+                                      :durable => opts[:durable])
+        @error_queue.bind(@error_exchange, :routing_key => '#')
+
+        # Finally, bind the worker queue to our requeue exchange
+        queue.bind(@requeue_exchange, :routing_key => '#')
+
+        @max_retries = @opts[:retry_max_times] || 5
+
+      end
+
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, props, msg, requeue = false)
+        if requeue
+          # This was explicitly rejected specifying it be requeued so we do not
+          # want it to pass through our retry logic.
+          @channel.reject(hdr.delivery_tag, requeue)
+        else
+          handle_retry(hdr, props, msg, :reject)
+        end
+      end
+
+
+      def error(hdr, props, msg, err)
+        handle_retry(hdr, props, msg, err)
+      end
+
+      def timeout(hdr, props, msg)
+        handle_retry(hdr, props, msg, :timeout)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+
+      # Helper logic for retry handling. This will reject the message if there
+      # are remaining retries left on it, otherwise it will publish it to the
+      # error exchange along with the reason.
+      # @param hdr [Bunny::DeliveryInfo]
+      # @param props [Bunny::MessageProperties]
+      # @param msg [String] The message
+      # @param reason [String, Symbol, Exception] Reason for the retry, included
+      #   in the JSON we put on the error exchange.
+      def handle_retry(hdr, props, msg, reason)
+        # +1 for the current attempt
+        num_attempts = failure_count(props[:headers]) + 1
+        if num_attempts <= @max_retries
+          # We call reject which will route the message to the
+          # x-dead-letter-exchange (ie. retry exchange) on the queue
+          Sneakers.logger.info do
+            "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{props[:headers]}"
+          end
+          @channel.reject(hdr.delivery_tag, false)
+          # TODO: metrics
+        else
+          # Retried more than the max times
+          # Publish the original message with the routing_key to the error exchange
+          Sneakers.logger.info do
+            "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
+          end
+          data = {
+            error: reason,
+            num_attempts: num_attempts,
+            failed_at: Time.now.iso8601,
+            payload: Base64.encode64(msg.to_s)
+          }.tap do |hash|
+            if reason.is_a?(Exception)
+              hash[:error_class] = reason.class
+              hash[:error_message] = "#{reason}"
+              if reason.backtrace
+                hash[:backtrace] = reason.backtrace.take(10).join(', ')
+              end
+            end
+          end.to_json
+          @error_exchange.publish(data, :routing_key => hdr.routing_key)
+          @channel.acknowledge(hdr.delivery_tag, false)
+          # TODO: metrics
+        end
+      end
+      private :handle_retry
+
+      # Uses the x-death header to determine the number of failures this job has
+      # seen in the past. This does not count the current failure. So for
+      # instance, the first time the job fails, this will return 0, the second
+      # time, 1, etc.
+      # @param headers [Hash] Hash of headers that Rabbit delivers as part of
+      #   the message
+      # @return [Integer] Count of number of failures.
+      def failure_count(headers)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          x_death_array = headers['x-death'].select do |x_death|
+            x_death['queue'] == @worker_queue_name
+          end
+          if x_death_array.count > 0 && x_death_array.first['count']
+            # Newer versions of RabbitMQ return headers with a count key
+            x_death_array.inject(0) {|sum, x_death| sum + x_death['count']}
+          else
+            # Older versions return a separate x-death header for each failure
+            x_death_array.count
+          end
+        end
+      end
+      private :failure_count
+
+      # Prefix all of our log messages so they are easier to find. We don't have
+      # the worker, so the next best thing is the queue name.
+      def log_prefix
+        "Maxretry handler [queue=#{@worker_queue_name}]"
+      end
+      private :log_prefix
+
+    end
+  end
+end

data/lib/sneakers/handlers/oneshot.rb

@@ -0,0 +1,30 @@
+module Sneakers
+  module Handlers
+    class Oneshot
+      def initialize(channel, queue, opts)
+        @channel = channel
+        @opts = opts
+      end
+
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, props, msg, requeue=false)
+        @channel.reject(hdr.delivery_tag, requeue)
+      end
+
+      def error(hdr, props, msg, err)
+        reject(hdr, props, msg)
+      end
+
+      def timeout(hdr, props, msg)
+        reject(hdr, props, msg)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+    end
+  end
+end

data/lib/sneakers/metrics/logging_metrics.rb

@@ -0,0 +1,16 @@
+module Sneakers
+  module Metrics
+    class LoggingMetrics
+      def increment(metric)
+        Sneakers.logger.info("INC: #{metric}")
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        Sneakers.logger.info("TIME: #{metric} #{Time.now - start}")
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/newrelic_metrics.rb

@@ -0,0 +1,37 @@
+module Sneakers
+  module Metrics
+    class NewrelicMetrics
+
+      def self.eagent(eagent = nil)
+        @eagent = eagent || @eagent
+      end
+
+      def initialize()
+        #@connection = conn
+      end
+
+      def increment(metric)
+        record_stat metric, 1
+      end
+
+      def record_stat(metric, num)
+        stats(metric).record_data_point(num)
+      rescue Exception => e
+        puts "NewrelicMetrics#record_stat: #{e}"
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        record_stat(metric, ((Time.now - start)*1000).floor)
+      end
+
+      def stats(metric)
+        metric.gsub! "\.", "\/"
+        NewrelicMetrics.eagent::Agent.get_stats("Custom/#{metric}")
+      end
+
+    end
+  end
+end
+

data/lib/sneakers/metrics/null_metrics.rb

@@ -0,0 +1,13 @@
+module Sneakers
+  module Metrics
+    class NullMetrics
+      def increment(metric)
+      end
+
+      def timing(metric, &block)
+        block.call
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/statsd_metrics.rb

@@ -0,0 +1,21 @@
+module Sneakers
+  module Metrics
+    class StatsdMetrics
+      def initialize(conn)
+        @connection = conn
+      end
+
+      def increment(metric)
+        @connection.increment(metric)
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        @connection.timing(metric, ((Time.now - start)*1000).floor)
+      end
+
+    end
+  end
+end
+

data/lib/sneakers/publisher.rb

@@ -0,0 +1,34 @@
+module Sneakers
+  class Publisher
+    def initialize(opts = {})
+      @mutex = Mutex.new
+      @opts = Sneakers::CONFIG.merge(opts)
+    end
+
+    def publish(msg, options = {})
+      @mutex.synchronize do
+        ensure_connection! unless connected?
+      end
+      to_queue = options.delete(:to_queue)
+      options[:routing_key] ||= to_queue
+      Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
+      @exchange.publish(msg, options)
+    end
+
+
+    attr_reader :exchange
+
+  private
+    def ensure_connection!
+      @bunny = Bunny.new(@opts[:amqp], heartbeat: @opts[:heartbeat], vhost: @opts[:vhost], :logger => Sneakers::logger)
+      @bunny.start
+      @channel = @bunny.create_channel
+      @exchange = @channel.exchange(@opts[:exchange], type: @opts[:exchange_type], durable: @opts[:durable])
+    end
+
+    def connected?
+      @bunny && @bunny.connected?
+    end
+  end
+end
+

data/lib/sneakers/queue.rb

@@ -0,0 +1,65 @@
+
+class Sneakers::Queue
+  attr_reader :name, :opts, :exchange
+
+  def initialize(name, opts)
+    @name = name
+    @opts = opts
+    @handler_klass = Sneakers::CONFIG[:handler]
+  end
+
+  #
+  # :exchange
+  # :heartbeat_interval
+  # :prefetch
+  # :durable
+  # :ack
+  #
+  def subscribe(worker)
+    #@bunny = Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], :heartbeat => @opts[:heartbeat], :logger => Sneakers::logger)
+    @bunny = @opts[:connection]
+    @bunny ||= Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], :heartbeat => @opts[:heartbeat], :logger => Sneakers::logger)
+    @bunny.start
+
+    @channel = @bunny.create_channel
+    @channel.prefetch(@opts[:prefetch])
+
+    exchange_name = @opts[:exchange]
+    @exchange = @channel.exchange(exchange_name,
+                                  :type => @opts[:exchange_type],
+                                  :durable => @opts[:durable])
+
+    routing_key = @opts[:routing_key] || @name
+    routing_keys = [*routing_key]
+
+    # TODO: get the arguments from the handler? Retry handler wants this so you
+    # don't have to line up the queue's dead letter argument with the exchange
+    # you'll create for retry.
+    queue_durable = @opts[:queue_durable].nil? ? @opts[:durable] : @opts[:queue_durable]
+    queue = @channel.queue(@name, :durable => queue_durable, :arguments => @opts[:arguments])
+
+    if exchange_name.length > 0
+      routing_keys.each do |key|
+        queue.bind(@exchange, :routing_key => key)
+      end
+    end
+
+    # NOTE: we are using the worker's options. This is necessary so the handler
+    # has the same configuration as the worker. Also pass along the exchange and
+    # queue in case the handler requires access to them (for things like binding
+    # retry queues, etc).
+    handler_klass = worker.opts[:handler] || Sneakers::CONFIG.fetch(:handler)
+    handler = handler_klass.new(@channel, queue, worker.opts)
+
+    @consumer = queue.subscribe(:block => false, :manual_ack => @opts[:ack]) do | delivery_info, metadata, msg |
+      worker.do_work(delivery_info, metadata, msg, handler)
+    end
+    nil
+  end
+
+  def unsubscribe
+    # XXX can we cancel bunny and channel too?
+    @consumer.cancel if @consumer
+    @consumer = nil
+  end
+end

data/lib/sneakers/runner.rb

@@ -0,0 +1,82 @@
+require 'serverengine'
+require 'sneakers/workergroup'
+
+module Sneakers
+  class Runner
+    def initialize(worker_classes, opts={})
+      @runnerconfig = RunnerConfig.new(worker_classes, opts)
+    end
+
+    def run
+      @se = ServerEngine.create(nil, WorkerGroup) { @runnerconfig.reload_config! }
+      @se.run
+    end
+
+    def stop
+      @se.stop
+    end
+  end
+
+
+  class RunnerConfig
+    def method_missing(meth, *args, &block)
+      if %w{ before_fork after_fork }.include? meth.to_s
+        @conf[meth] = block
+      elsif %w{ workers start_worker_delay amqp }.include? meth.to_s
+        @conf[meth] = args.first
+      else
+        super
+      end
+    end
+
+    def initialize(worker_classes, opts)
+      @worker_classes = worker_classes
+      @conf = opts
+    end
+
+    def to_h
+      @conf
+    end
+
+
+    def reload_config!
+      Sneakers.logger.warn("Loading runner configuration...")
+      config_file = Sneakers::CONFIG[:runner_config_file]
+
+      if config_file
+        begin
+          instance_eval(File.read(config_file), config_file)
+          Sneakers.logger.info("Loading config with file: #{config_file}")
+        rescue
+          Sneakers.logger.error("Cannot load from file '#{config_file}', #{$!}")
+        end
+      end
+
+      config = make_serverengine_config
+
+      [:before_fork, :after_fork].each do | hook |
+        Sneakers::CONFIG[:hooks][hook] = config.delete(hook) if config[hook]
+      end
+
+
+      Sneakers.logger.info("New configuration: #{config.inspect}")
+      config
+    end
+
+  private
+  def make_serverengine_config
+      # From Sneakers#setup_general_logger, there's support for a Logger object
+      # in CONFIG[:log].  However, serverengine takes an object in :logger.
+      # Pass our logger object so there's no issue about sometimes passing a
+      # file and sometimes an object.
+      without_log = Sneakers::CONFIG.merge(@conf)
+      without_log.delete(:log)
+      Sneakers::CONFIG.merge(@conf).merge({
+        :logger => Sneakers.logger,
+        :worker_type => 'process',
+        :worker_classes => @worker_classes
+      })
+    end
+  end
+
+end