kicks 3.0.0.pre

data/lib/sneakers/handlers/maxretry.rb

@@ -0,0 +1,219 @@
+require 'base64'
+require 'json'
+
+module Sneakers
+  module Handlers
+    #
+    # Maxretry uses dead letter policies on Rabbitmq to requeue and retry
+    # messages after failure (rejections and errors). When the maximum
+    # number of retries is reached it will put the message on an error queue.
+    # This handler will only retry at the queue level. To accomplish that, the
+    # setup is a bit complex.
+    #
+    # Input:
+    #   worker_exchange (eXchange)
+    #   worker_queue (Queue)
+    # We create:
+    #   worker_queue-retry - (X) where we setup the worker queue to dead-letter.
+    #   worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to
+    #                        worker_queue-retry-requeue.
+    #   worker_queue-error - (X) where to send max-retry failures
+    #   worker_queue-error - (Q) bound to worker_queue-error.
+    #   worker_queue-retry-requeue - (X) exchange to bind worker_queue to for
+    #                                requeuing directly to the worker_queue.
+    #
+    # This requires that you setup arguments to the worker queue to line up the
+    # dead letter queue. See the example for more information.
+    #
+    # Many of these can be override with options:
+    # - retry_exchange - sets retry exchange & queue
+    # - retry_error_exchange - sets error exchange and queue
+    # - retry_requeue_exchange - sets the exchange created to re-queue things
+    #   back to the worker queue.
+    #
+    class Maxretry
+
+      def initialize(channel, queue, opts)
+        @worker_queue_name = queue.name
+        Sneakers.logger.debug do
+          "#{log_prefix} creating handler, opts=#{opts}"
+        end
+
+        @channel = channel
+        @opts = opts
+
+        # Construct names, defaulting where suitable
+        retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry"
+        error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error"
+        requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue"
+        retry_routing_key = @opts[:retry_routing_key] || "#"
+
+        # Create the exchanges
+        @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name|
+          Sneakers.logger.debug { "#{log_prefix} creating exchange=#{name}" }
+          @channel.exchange(name,
+                            :type => 'topic',
+                            :durable => exchange_durable?)
+        end
+
+        # Create the queues and bindings
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}"
+        end
+        @retry_queue = @channel.queue(retry_name,
+                                     :durable => queue_durable?,
+                                     :arguments => {
+                                       :'x-dead-letter-exchange' => requeue_name,
+                                       :'x-message-ttl' => @opts[:retry_timeout] || 60000
+                                     })
+        @retry_queue.bind(@retry_exchange, :routing_key => '#')
+
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{error_name}"
+        end
+        @error_queue = @channel.queue(error_name,
+                                      :durable => queue_durable?)
+        @error_queue.bind(@error_exchange, :routing_key => '#')
+
+        # Finally, bind the worker queue to our requeue exchange
+        queue.bind(@requeue_exchange, :routing_key => retry_routing_key)
+
+        @max_retries = @opts[:retry_max_times] || 5
+
+      end
+
+      def self.configure_queue(name, opts)
+        retry_name = opts.fetch(:retry_exchange, "#{name}-retry")
+        opt_args = opts[:queue_options][:arguments] ? opts[:queue_options][:arguments].inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} : {}
+        opts[:queue_options][:arguments] = { :'x-dead-letter-exchange' => retry_name }.merge(opt_args)
+        opts[:queue_options]
+      end
+
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, props, msg, requeue = false)
+        if requeue
+          # This was explicitly rejected specifying it be requeued so we do not
+          # want it to pass through our retry logic.
+          @channel.reject(hdr.delivery_tag, requeue)
+        else
+          handle_retry(hdr, props, msg, :reject)
+        end
+      end
+
+
+      def error(hdr, props, msg, err)
+        handle_retry(hdr, props, msg, err)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+
+      # Helper logic for retry handling. This will reject the message if there
+      # are remaining retries left on it, otherwise it will publish it to the
+      # error exchange along with the reason.
+      # @param hdr [Bunny::DeliveryInfo]
+      # @param props [Bunny::MessageProperties]
+      # @param msg [String] The message
+      # @param reason [String, Symbol, Exception] Reason for the retry, included
+      #   in the JSON we put on the error exchange.
+      def handle_retry(hdr, props, msg, reason)
+        # +1 for the current attempt
+        num_attempts = failure_count(props[:headers]) + 1
+        if num_attempts <= @max_retries
+          # We call reject which will route the message to the
+          # x-dead-letter-exchange (ie. retry exchange) on the queue
+          Sneakers.logger.info do
+            "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{props[:headers]}"
+          end
+          @channel.reject(hdr.delivery_tag, false)
+          # TODO: metrics
+        else
+          # Retried more than the max times
+          # Publish the original message with the routing_key to the error exchange
+          Sneakers.logger.info do
+            "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
+          end
+          data = {
+            error: reason.to_s,
+            num_attempts: num_attempts,
+            failed_at: Time.now.iso8601,
+            properties: props.to_hash
+          }.tap do |hash|
+            if reason.is_a?(Exception)
+              hash[:error_class] = reason.class.to_s
+              hash[:error_message] = "#{reason}"
+              if reason.backtrace
+                hash[:backtrace] = reason.backtrace.take(10)
+              end
+            end
+          end
+
+          # Preserve retry log in a list
+          if retry_info = props[:headers]['retry_info']
+            old_retry0 = JSON.parse(retry_info) rescue {error: "Failed to parse retry info"}
+            old_retry  = Array(old_retry0)
+            # Prevent old retry from nesting
+            data[:properties][:headers].delete('retry_info')
+            data = old_retry.unshift(data)
+          end
+
+          @error_exchange.publish(msg, {
+            routing_key: hdr.routing_key,
+            headers: {
+              retry_info: data.to_json
+            }
+          })
+          @channel.acknowledge(hdr.delivery_tag, false)
+          # TODO: metrics
+        end
+      end
+      private :handle_retry
+
+      # Uses the x-death header to determine the number of failures this job has
+      # seen in the past. This does not count the current failure. So for
+      # instance, the first time the job fails, this will return 0, the second
+      # time, 1, etc.
+      # @param headers [Hash] Hash of headers that Rabbit delivers as part of
+      #   the message
+      # @return [Integer] Count of number of failures.
+      def failure_count(headers)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          x_death_array = headers['x-death'].select do |x_death|
+            x_death['queue'] == @worker_queue_name
+          end
+          if x_death_array.count > 0 && x_death_array.first['count']
+            # Newer versions of RabbitMQ return headers with a count key
+            x_death_array.inject(0) {|sum, x_death| sum + x_death['count']}
+          else
+            # Older versions return a separate x-death header for each failure
+            x_death_array.count
+          end
+        end
+      end
+      private :failure_count
+
+      # Prefix all of our log messages so they are easier to find. We don't have
+      # the worker, so the next best thing is the queue name.
+      def log_prefix
+        "Maxretry handler [queue=#{@worker_queue_name}]"
+      end
+      private :log_prefix
+
+      private
+
+      def queue_durable?
+        @opts.fetch(:queue_options, {}).fetch(:durable, false)
+      end
+
+      def exchange_durable?
+        queue_durable?
+      end
+    end
+  end
+end

data/lib/sneakers/handlers/oneshot.rb

@@ -0,0 +1,26 @@
+module Sneakers
+  module Handlers
+    class Oneshot
+      def initialize(channel, queue, opts)
+        @channel = channel
+        @opts = opts
+      end
+
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, props, msg, requeue=false)
+        @channel.reject(hdr.delivery_tag, requeue)
+      end
+
+      def error(hdr, props, msg, err)
+        reject(hdr, props, msg)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+    end
+  end
+end

data/lib/sneakers/metrics/logging_metrics.rb

@@ -0,0 +1,16 @@
+module Sneakers
+  module Metrics
+    class LoggingMetrics
+      def increment(metric)
+        Sneakers.logger.info("INC: #{metric}")
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        Sneakers.logger.info("TIME: #{metric} #{Time.now - start}")
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/newrelic_metrics.rb

@@ -0,0 +1,32 @@
+module Sneakers
+  module Metrics
+    class NewrelicMetrics
+
+      def self.eagent(eagent = nil)
+        @eagent = eagent || @eagent
+      end
+
+      def initialize()
+        #@connection = conn
+      end
+
+      def increment(metric)
+        record_stat metric, 1
+      end
+
+      def record_stat(metric, num)
+        metric_name = "Custom/#{metric.gsub("\.", "\/")}"
+        NewrelicMetrics.eagent::Agent.record_metric(metric_name, num)
+      rescue Exception => e
+        puts "NewrelicMetrics#record_stat: #{e}"
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        record_stat(metric, ((Time.now - start)*1000).floor)
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/null_metrics.rb

@@ -0,0 +1,13 @@
+module Sneakers
+  module Metrics
+    class NullMetrics
+      def increment(metric)
+      end
+
+      def timing(metric, &block)
+        block.call
+      end
+    end
+  end
+end
+

data/lib/sneakers/metrics/statsd_metrics.rb

@@ -0,0 +1,21 @@
+module Sneakers
+  module Metrics
+    class StatsdMetrics
+      def initialize(conn)
+        @connection = conn
+      end
+
+      def increment(metric)
+        @connection.increment(metric)
+      end
+
+      def timing(metric, &block)
+        start = Time.now
+        block.call
+        @connection.timing(metric, ((Time.now - start)*1000).floor)
+      end
+
+    end
+  end
+end
+

data/lib/sneakers/middleware/config.rb

@@ -0,0 +1,23 @@
+module Sneakers
+  module Middleware
+    class Config
+      def self.use(klass, args)
+        middlewares << { class: klass, args: args }
+      end
+
+      def self.delete(klass)
+        middlewares.reject! { |el| el[:class] == klass }
+      end
+
+      def self.to_a
+        middlewares
+      end
+
+      def self.middlewares
+        @middlewares ||= []
+      end
+
+      private_class_method :middlewares
+    end
+  end
+end

data/lib/sneakers/publisher.rb

@@ -0,0 +1,49 @@
+module Sneakers
+  class Publisher
+
+    attr_reader :exchange, :channel
+
+    def initialize(opts = {})
+      @mutex = Mutex.new
+      @opts = Sneakers::CONFIG.merge(opts)
+      # If we've already got a bunny object, use it.  This allows people to
+      # specify all kinds of options we don't need to know about (e.g. for ssl).
+      @bunny = @opts[:connection]
+    end
+
+    def publish(msg, options = {})
+      ensure_connection!
+      to_queue = options.delete(:to_queue)
+      options[:routing_key] ||= to_queue
+      Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
+      serialized_msg = Sneakers::ContentType.serialize(msg, options[:content_type])
+      encoded_msg = Sneakers::ContentEncoding.encode(serialized_msg, options[:content_encoding])
+      @exchange.publish(encoded_msg, options)
+    end
+
+    def ensure_connection!
+      @mutex.synchronize do
+        connect! unless connected?
+      end
+    end
+
+  private
+    def connect!
+      @bunny ||= create_bunny_connection
+      @bunny.start
+      @channel = @bunny.create_channel
+      @exchange = @channel.exchange(@opts[:exchange], **@opts[:exchange_options])
+    end
+
+    def connected?
+      @bunny && @bunny.connected? && channel
+    end
+
+    def create_bunny_connection
+      Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
+                              :heartbeat => @opts[:heartbeat],
+                              :properties => @opts.fetch(:properties, {}),
+                              :logger => Sneakers::logger)
+    end
+  end
+end

data/lib/sneakers/queue.rb

@@ -0,0 +1,87 @@
+
+class Sneakers::Queue
+  attr_reader :name, :opts, :exchange, :channel
+
+  def initialize(name, opts)
+    @name = name
+    @opts = opts
+    @handler_klass = Sneakers::CONFIG[:handler]
+  end
+
+  #
+  # :exchange
+  # :heartbeat_interval
+  # :prefetch
+  # :durable
+  # :ack
+  #
+  def subscribe(worker)
+    # If we've already got a bunny object, use it.  This allows people to
+    # specify all kinds of options we don't need to know about (e.g. for ssl).
+    @bunny = @opts[:connection]
+    @bunny ||= create_bunny_connection
+    @bunny.start
+
+    @channel = @bunny.create_channel
+    @channel.prefetch(@opts[:prefetch])
+
+    exchange_name = @opts[:exchange]
+    @exchange = @channel.exchange(exchange_name, **@opts[:exchange_options])
+
+    routing_key = @opts[:routing_key] || @name
+    routing_keys = [*routing_key]
+
+    handler_klass = worker.opts[:handler] || Sneakers::CONFIG.fetch(:handler)
+    # Configure options if needed
+    if handler_klass.respond_to?(:configure_queue)
+      @opts[:queue_options] = handler_klass.configure_queue(@name, @opts)
+    end
+
+    queue = @channel.queue(@name, **@opts[:queue_options])
+
+    if exchange_name.length > 0
+      routing_keys.each do |key|
+        if @opts[:bind_arguments]
+          queue.bind(@exchange, routing_key: key, arguments: @opts[:bind_arguments])
+        else
+          queue.bind(@exchange, routing_key: key)
+        end
+      end
+    end
+
+    # NOTE: we are using the worker's options. This is necessary so the handler
+    # has the same configuration as the worker. Also pass along the exchange and
+    # queue in case the handler requires access to them (for things like binding
+    # retry queues, etc).
+    handler = handler_klass.new(@channel, queue, worker.opts)
+
+    @consumer = queue.subscribe(block: false, manual_ack: @opts[:ack]) do | delivery_info, metadata, msg |
+      worker.do_work(delivery_info, metadata, msg, handler)
+    end
+    nil
+  end
+
+  def unsubscribe
+    return unless @consumer
+
+    # TODO: should we simply close the channel here?
+    Sneakers.logger.info("Queue: will try to cancel consumer #{@consumer.inspect}")
+    cancel_ok = @consumer.cancel
+    if cancel_ok
+      Sneakers.logger.info "Queue: consumer #{cancel_ok.consumer_tag} cancelled"
+      @consumer = nil
+    else
+      Sneakers.logger.warn "Queue: could not cancel consumer #{@consumer.inspect}"
+      sleep(1)
+      unsubscribe
+    end
+  end
+
+  def create_bunny_connection
+    Bunny.new(@opts[:amqp], { vhost: @opts[:vhost],
+                            heartbeat: @opts[:heartbeat],
+                            properties: @opts.fetch(:properties, {}),
+                            logger: Sneakers::logger })
+  end
+  private :create_bunny_connection
+end

data/lib/sneakers/runner.rb

@@ -0,0 +1,91 @@
+require 'serverengine'
+require 'sneakers/workergroup'
+
+module Sneakers
+  class Runner
+    def initialize(worker_classes, opts={})
+      @runnerconfig = RunnerConfig.new(worker_classes, opts)
+    end
+
+    def run
+      @se = ServerEngine.create(nil, WorkerGroup) { @runnerconfig.reload_config! }
+      @se.run
+    end
+
+    def stop(stop_graceful=true)
+      @se.stop(stop_graceful)
+    end
+  end
+
+
+  class RunnerConfig
+    def method_missing(meth, *args, &block)
+      if %w{ before_fork after_fork }.include? meth.to_s
+        @conf[meth] = block
+      elsif %w{ workers start_worker_delay amqp }.include? meth.to_s
+        @conf[meth] = args.first
+      else
+        super
+      end
+    end
+
+    def initialize(worker_classes, opts)
+      @worker_classes = worker_classes
+      @conf = opts
+    end
+
+    def to_h
+      @conf
+    end
+
+
+    def reload_config!
+      Sneakers.logger.info("Loading runner configuration...")
+      config_file = Sneakers::CONFIG[:runner_config_file]
+
+      if config_file
+        begin
+          instance_eval(File.read(config_file), config_file)
+          Sneakers.logger.info("Loading config with file: #{config_file}")
+        rescue
+          Sneakers.logger.error("Cannot load from file '#{config_file}', #{$!}")
+        end
+      end
+
+      config = make_serverengine_config
+
+      [:before_fork, :after_fork].each do | hook |
+        Sneakers::CONFIG[:hooks][hook] = config.delete(hook) if config[hook]
+      end
+
+      Sneakers.logger.debug("New configuration: #{config.inspect}")
+      config
+    end
+
+    private
+
+    def make_serverengine_config
+      # From Sneakers#setup_general_logger, there's support for a Logger object
+      # in CONFIG[:log].  However, serverengine takes an object in :logger.
+      # Pass our logger object so there's no issue about sometimes passing a
+      # file and sometimes an object.
+      serverengine_config =  Sneakers::CONFIG.merge(@conf)
+      serverengine_config.merge!(
+        :logger => Sneakers.logger,
+        :log_level => Sneakers.logger.level,
+        :worker_type => 'process',
+        :worker_classes => @worker_classes,
+
+        # Turning off serverengine internal logging infra, causes
+        # livelock and hang.
+        # see https://github.com/jondot/sneakers/issues/153
+        :log_stdout => false,
+        :log_stderr => false
+      )
+      serverengine_config.delete(:log)
+
+      serverengine_config
+    end
+  end
+
+end

data/lib/sneakers/spawner.rb

@@ -0,0 +1,30 @@
+require 'yaml'
+require 'erb'
+
+module Sneakers
+  class Spawner
+    def self.spawn
+      worker_group_config_file = ENV['WORKER_GROUP_CONFIG'] || './config/sneaker_worker_groups.yml'
+      unless File.exist?(worker_group_config_file)
+        puts 'No worker group file found.'
+        puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
+        Kernel.exit(1)
+      end
+      @pids = []
+      @exec_string = 'bundle exec rake sneakers:run'
+      worker_config = YAML.load(ERB.new(File.read(worker_group_config_file)).result)
+      worker_config.keys.each do |group_name|
+        workers = worker_config[group_name]['classes']
+        workers = workers.join ',' if workers.is_a?(Array)
+        @pids << fork do
+          @exec_hash = { 'WORKERS' => workers, 'WORKER_COUNT' => worker_config[group_name]['workers'].to_s }
+          Kernel.exec(@exec_hash, @exec_string)
+        end
+      end
+      %w[TERM USR1 HUP USR2].each do |signal|
+        Signal.trap(signal) { @pids.each { |pid| Process.kill(signal, pid) } }
+      end
+      Process.waitall
+    end
+  end
+end

data/lib/sneakers/support/production_formatter.rb

@@ -0,0 +1,11 @@
+require 'time'
+module Sneakers
+  module Support
+    class ProductionFormatter < Logger::Formatter
+        def self.call(severity, time, program_name, message)
+          "#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
+        end
+    end
+  end
+end
+

data/lib/sneakers/support/utils.rb

@@ -0,0 +1,18 @@
+class Sneakers::Utils
+  def self.make_worker_id(namespace)
+    "worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}"  # jid, worker id. include date.
+  end
+  def self.parse_workers(workerstring)
+    missing_workers = []
+    workers = (workerstring || '').split(',').map do |k|
+      begin
+        w = Kernel.const_get(k)
+      rescue
+        missing_workers << k
+      end
+      w
+    end.compact
+
+    [workers, missing_workers]
+  end
+end