sneakers 0.1.1.pre → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9d39e1dc0f9c583d63d7ba5d6cf8078efe94f93f
+  data.tar.gz: 7bf1d48e4a492d454db3afc7474b9053adc05531
+SHA512:
+  metadata.gz: d7c50e77e8a9de0e4ebed4d6120e78c5b5438d4d430ed776a686a69e6a288f000a4a1f40cf9a0bb5ab861bc7e5cb8fc93a5d8241301f15888b5ac3682447ae58
+  data.tar.gz: 80942f71ebc85d9365ada65413ec16f0f3444631e568f930de6279d6924301f74745336449e72a7da1963b01e0945fa0d155654164b7f3af44e22c9b6851def0

data/.gitignore

@@ -1,6 +1,9 @@
 sneakers.yaml
+Gemfile.lock
 *.log
 sneakers.pid
 pkg/
 coverage/
 tmp/
+.ruby-version
+.ruby-gemset

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Dotan Nahum
+Copyright (c) 2013-2014 Dotan Nahum
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -7,4 +7,5 @@ Rake::TestTask.new do |t|
   t.test_files = FileList['spec/**/*_spec.rb']
 end
 
+task default: :test
 

data/examples/max_retry_handler.rb

@@ -0,0 +1,78 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'sneakers/handlers/maxretry'
+require 'logger'
+
+Sneakers.configure(:handler => Sneakers::Handlers::Maxretry,
+                   :workers => 1,
+                   :threads => 1,
+                   :prefetch => 1,
+                   :exchange => 'sneakers',
+                   :exchange_type => 'topic',
+                   :routing_key => ['#', 'something'],
+                   :durable => true,
+                   )
+Sneakers.logger.level = Logger::DEBUG
+
+WORKER_OPTIONS = {
+  :ack => true,
+  :threads => 1,
+  :prefetch => 1,
+  :timeout_job_after => 60,
+  :heartbeat => 5,
+  :retry_timeout => 5000
+}
+
+# Example of how to write a retry worker. If your rabbit system is empty, then
+# you must run this twice. Once to setup the exchanges, queues and bindings a
+# second time to have the sent message end up on the downloads queue.
+#
+# Run this via:
+#   bundle exec ruby examples/max_retry_handler.rb
+#
+class MaxRetryWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+      WORKER_OPTIONS.merge({
+                             :arguments => {
+                               :'x-dead-letter-exchange' => 'downloads-retry'
+                             },
+                           })
+
+  def work(msg)
+    logger.info("MaxRetryWorker rejecting msg: #{msg.inspect}")
+
+    # We always want to reject to see if we do the proper timeout
+    reject!
+  end
+end
+
+# Example of a worker on the same exchange that does not fail, so it should only
+# see the message once.
+class SucceedingWorker
+  include Sneakers::Worker
+  from_queue 'uploads',
+      WORKER_OPTIONS.merge({
+                             :arguments => {
+                               :'x-dead-letter-exchange' => 'uploads-retry'
+                             },
+                           })
+
+  def work(msg)
+    logger.info("SucceedingWorker succeeding on msg: #{msg.inspect}")
+    ack!
+  end
+end
+
+messages = 1
+puts "feeding messages in"
+messages.times {
+  Sneakers.publish(" -- message -- ",
+                   :to_queue => 'anywhere',
+                   :persistence => true)
+}
+puts "done"
+
+r = Sneakers::Runner.new([MaxRetryWorker, SucceedingWorker])
+r.run

data/examples/newrelic_metrics_worker.rb

@@ -0,0 +1,40 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'sneakers/metrics/newrelic_metrics'
+require 'open-uri'
+require 'nokogiri'
+require 'newrelic_rpm'
+
+# With this configuration will send two types of data to newrelic server:
+# 1. Transaction data which you would see under  'Applications'
+# 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts
+#
+# You should have newrelic.yml in the 'config' folder with the proper account settings
+
+Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic
+Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new
+
+class MetricsWorker
+  include Sneakers::Worker
+  include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    logger.info "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+  
+  add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
+
+end
+
+
+r = Sneakers::Runner.new([ MetricsWorker ])
+r.run
+
+
+
+

data/lib/sneakers/cli.rb

@@ -66,11 +66,11 @@ module Sneakers
 
       r = Sneakers::Runner.new(workers)
 
-      pid = Sneakers::Config[:pid_path]
+      pid = Sneakers::CONFIG[:pid_path]
 
       say SNEAKERS
       say "Workers ....: #{em workers.join(', ')}"
-      say "Log ........: #{em (Sneakers::Config[:log] == STDOUT ? 'Console' : Sneakers::Config[:log]) }"
+      say "Log ........: #{em (Sneakers::CONFIG[:log] == STDOUT ? 'Console' : Sneakers::CONFIG[:log]) }"
       say "PID ........: #{em pid}"
       say ""
       say (" "*31)+"Process control"
@@ -86,7 +86,7 @@ module Sneakers
 
       if options[:debug]
         say "==== configuration ==="
-        say Sneakers::Config.inspect
+        say Sneakers::CONFIG.inspect
         say "======================"
       end
 

data/lib/sneakers/concerns/logging.rb

@@ -23,7 +23,7 @@ module Sneakers
           else
             @logger = Logger.new(STDOUT)
             @logger.level = Logger::INFO
-            @logger.formatter = ProductionFormatter
+            @logger.formatter = Sneakers::Support::ProductionFormatter
           end
         end
       end

data/lib/sneakers/configuration.rb

@@ -0,0 +1,47 @@
+module Sneakers
+  class Configuration
+
+    extend Forwardable
+    def_delegators :@hash, :to_hash, :[], :[]=, :merge!, :==, :fetch, :delete
+
+    DEFAULTS = {
+      # runner
+      :runner_config_file => nil,
+      :metrics            => nil,
+      :daemonize          => false,
+      :start_worker_delay => 0.2,
+      :workers            => 4,
+      :log                => STDOUT,
+      :pid_path           => 'sneakers.pid',
+
+      # workers
+      :timeout_job_after  => 5,
+      :prefetch           => 10,
+      :threads            => 10,
+      :durable            => true,
+      :ack                => true,
+      :heartbeat          => 2,
+      :exchange           => 'sneakers',
+      :exchange_type      => :direct,
+      :hooks              => {}
+    }.freeze
+
+
+    def initialize
+      clear
+    end
+
+    def clear
+      @hash = DEFAULTS.dup
+      @hash[:amqp]  = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672')
+      @hash[:vhost] = AMQ::Settings.parse_amqp_url(@hash[:amqp]).fetch(:vhost, '/')
+    end
+
+    def merge(hash)
+      instance = self.class.new
+      instance.merge! to_hash
+      instance.merge! hash
+      instance
+    end
+  end
+end

data/lib/sneakers/handlers/maxretry.rb

@@ -0,0 +1,183 @@
+require 'base64'
+require 'json'
+
+module Sneakers
+  module Handlers
+    #
+    # Maxretry uses dead letter policies on Rabbitmq to requeue and retry
+    # messages after failure (rejections, errors and timeouts). When the maximum
+    # number of retries is reached it will put the message on an error queue.
+    # This handler will only retry at the queue level. To accomplish that, the
+    # setup is a bit complex.
+    #
+    # Input:
+    #   worker_exchange (eXchange)
+    #   worker_queue (Queue)
+    # We create:
+    #   worker_queue-retry - (X) where we setup the worker queue to dead-letter.
+    #   worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to
+    #                        worker_queue-retry-requeue.
+    #   worker_queue-error - (X) where to send max-retry failures
+    #   worker_queue-error - (Q) bound to worker_queue-error.
+    #   worker_queue-retry-requeue - (X) exchange to bind worker_queue to for
+    #                                requeuing directly to the worker_queue.
+    #
+    # This requires that you setup arguments to the worker queue to line up the
+    # dead letter queue. See the example for more information.
+    #
+    # Many of these can be override with options:
+    # - retry_exchange - sets retry exchange & queue
+    # - retry_error_exchange - sets error exchange and queue
+    # - retry_requeue_exchange - sets the exchange created to re-queue things
+    #   back to the worker queue.
+    #
+    class Maxretry
+
+      def initialize(channel, queue, opts)
+        @worker_queue_name = queue.name
+        Sneakers.logger.debug do
+          "#{log_prefix} creating handler, opts=#{opts}"
+        end
+
+        @channel = channel
+        @opts = opts
+
+        # Construct names, defaulting where suitable
+        retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry"
+        error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error"
+        requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue"
+
+        # Create the exchanges
+        @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name|
+          Sneakers.logger.debug { "#{log_prefix} creating exchange=#{name}" }
+          @channel.exchange(name,
+                            :type => 'topic',
+                            :durable => opts[:durable])
+        end
+
+        # Create the queues and bindings
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}"
+        end
+        @retry_queue = @channel.queue(retry_name,
+                                     :durable => opts[:durable],
+                                     :arguments => {
+                                       :'x-dead-letter-exchange' => requeue_name,
+                                       :'x-message-ttl' => @opts[:retry_timeout] || 60000
+                                     })
+        @retry_queue.bind(@retry_exchange, :routing_key => '#')
+
+        Sneakers.logger.debug do
+          "#{log_prefix} creating queue=#{error_name}"
+        end
+        @error_queue = @channel.queue(error_name,
+                                      :durable => opts[:durable])
+        @error_queue.bind(@error_exchange, :routing_key => '#')
+
+        # Finally, bind the worker queue to our requeue exchange
+        queue.bind(@requeue_exchange, :routing_key => '#')
+
+        @max_retries = @opts[:retry_max_times] || 5
+
+      end
+
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, props, msg, requeue = false)
+        if requeue
+          # This was explicitly rejected specifying it be requeued so we do not
+          # want it to pass through our retry logic.
+          @channel.reject(hdr.delivery_tag, requeue)
+        else
+          handle_retry(hdr, props, msg, :reject)
+        end
+      end
+
+
+      def error(hdr, props, msg, err)
+        handle_retry(hdr, props, msg, err)
+      end
+
+      def timeout(hdr, props, msg)
+        handle_retry(hdr, props, msg, :timeout)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+
+      # Helper logic for retry handling. This will reject the message if there
+      # are remaining retries left on it, otherwise it will publish it to the
+      # error exchange along with the reason.
+      # @param hdr [Bunny::DeliveryInfo]
+      # @param props [Bunny::MessageProperties]
+      # @param msg [String] The message
+      # @param reason [String, Symbol, Exception] Reason for the retry, included
+      #   in the JSON we put on the error exchange.
+      def handle_retry(hdr, props, msg, reason)
+        # +1 for the current attempt
+        num_attempts = failure_count(props[:headers]) + 1
+        if num_attempts <= @max_retries
+          # We call reject which will route the message to the
+          # x-dead-letter-exchange (ie. retry exchange) on the queue
+          Sneakers.logger.info do
+            "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{props[:headers]}"
+          end
+          @channel.reject(hdr.delivery_tag, false)
+          # TODO: metrics
+        else
+          # Retried more than the max times
+          # Publish the original message with the routing_key to the error exchange
+          Sneakers.logger.info do
+            "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
+          end
+          data = {
+            error: reason,
+            num_attempts: num_attempts,
+            failed_at: Time.now.iso8601,
+            payload: Base64.encode64(msg.to_s)
+          }.tap do |hash|
+            if reason.is_a?(Exception)
+              hash[:error_class] = reason.class
+              if reason.backtrace
+                hash[:backtrace] = reason.backtrace.take(10).join(', ')
+              end
+            end
+          end.to_json
+          @error_exchange.publish(data, :routing_key => hdr.routing_key)
+          @channel.acknowledge(hdr.delivery_tag, false)
+          # TODO: metrics
+        end
+      end
+      private :handle_retry
+
+      # Uses the x-death header to determine the number of failures this job has
+      # seen in the past. This does not count the current failure. So for
+      # instance, the first time the job fails, this will return 0, the second
+      # time, 1, etc.
+      # @param headers [Hash] Hash of headers that Rabbit delivers as part of
+      #   the message
+      # @return [Integer] Count of number of failures.
+      def failure_count(headers)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          headers['x-death'].select do |x_death|
+            x_death['queue'] == @worker_queue_name
+          end.count
+        end
+      end
+      private :failure_count
+
+      # Prefix all of our log messages so they are easier to find. We don't have
+      # the worker, so the next best thing is the queue name.
+      def log_prefix
+        "Maxretry handler [queue=#{@worker_queue_name}]"
+      end
+      private :log_prefix
+
+    end
+  end
+end

data/lib/sneakers/handlers/oneshot.rb

@@ -1,27 +1,28 @@
 module Sneakers
   module Handlers
     class Oneshot
-      def initialize(channel)
+      def initialize(channel, queue, opts)
         @channel = channel
+        @opts = opts
       end
 
-      def acknowledge(tag)
-        @channel.acknowledge(tag, false)
+      def acknowledge(hdr, props, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
       end
 
-      def reject(tag, requeue=false)
-        @channel.reject(tag, requeue)
+      def reject(hdr, props, msg, requeue=false)
+        @channel.reject(hdr.delivery_tag, requeue)
       end
 
-      def error(tag, err)
-        reject(tag)
+      def error(hdr, props, msg, err)
+        reject(hdr, props, msg)
       end
 
-      def timeout(tag)
-        reject(tag)
+      def timeout(hdr, props, msg)
+        reject(hdr, props, msg)
       end
 
-      def noop(tag)
+      def noop(hdr, props, msg)
 
       end
     end

data/lib/sneakers/publisher.rb

@@ -1,28 +1,29 @@
 module Sneakers
   class Publisher
-    attr_accessor :exchange
-
-    def initialize(opts={})
+    def initialize(opts = {})
       @mutex = Mutex.new
-      @opts = Sneakers::Config.merge(opts)
+      @opts = Sneakers::CONFIG.merge(opts)
     end
 
-    def publish(msg, routing)
+    def publish(msg, options = {})
       @mutex.synchronize do
         ensure_connection! unless connected?
       end
-      Sneakers.logger.info("publishing <#{msg}> to [#{routing[:to_queue]}]")
-      @exchange.publish(msg, :routing_key => routing[:to_queue])
+      to_queue = options.delete(:to_queue)
+      options[:routing_key] ||= to_queue
+      Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
+      @exchange.publish(msg, options)
     end
 
+    private
 
-  private
+    attr_reader :exchange
 
     def ensure_connection!
-      @bunny = Bunny.new(@opts[:amqp], :heartbeat => @opts[:heartbeat])
+      @bunny = Bunny.new(@opts[:amqp], heartbeat: @opts[:heartbeat], vhost: @opts[:vhost], :logger => Sneakers::logger)
       @bunny.start
       @channel = @bunny.create_channel
-      @exchange = @channel.exchange(@opts[:exchange], :type => @opts[:exchange_type], :durable => @opts[:durable])
+      @exchange = @channel.exchange(@opts[:exchange], type: @opts[:exchange_type], durable: @opts[:durable])
     end
 
     def connected?

data/lib/sneakers/queue.rb

@@ -5,7 +5,7 @@ class Sneakers::Queue
   def initialize(name, opts)
     @name = name
     @opts = opts
-    @handler_klass = Sneakers::Config[:handler]
+    @handler_klass = Sneakers::CONFIG[:handler]
   end
 
   #
@@ -16,7 +16,7 @@ class Sneakers::Queue
   # :ack
   #
   def subscribe(worker)
-    @bunny = Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], :heartbeat => @opts[:heartbeat])
+    @bunny = Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], :heartbeat => @opts[:heartbeat], :logger => Sneakers::logger)
     @bunny.start
 
     @channel = @bunny.create_channel
@@ -26,19 +26,28 @@ class Sneakers::Queue
                                   :type => @opts[:exchange_type],
                                   :durable => @opts[:durable])
 
-    handler = @handler_klass.new(@channel)
-
     routing_key = @opts[:routing_key] || @name
     routing_keys = [*routing_key]
 
-    queue = @channel.queue(@name, :durable => @opts[:durable])
+    # TODO: get the arguments from the handler? Retry handler wants this so you
+    # don't have to line up the queue's dead letter argument with the exchange
+    # you'll create for retry.
+    queue_durable = @opts[:queue_durable].nil? ? @opts[:durable] : @opts[:queue_durable]
+    queue = @channel.queue(@name, :durable => queue_durable, :arguments => @opts[:arguments])
 
     routing_keys.each do |key|
       queue.bind(@exchange, :routing_key => key)
     end
 
-    @consumer = queue.subscribe(:block => false, :ack => @opts[:ack]) do | hdr, props, msg | 
-      worker.do_work(hdr, props, msg, handler)
+    # NOTE: we are using the worker's options. This is necessary so the handler
+    # has the same configuration as the worker. Also pass along the exchange and
+    # queue in case the handler requires access to them (for things like binding
+    # retry queues, etc).
+    handler_klass = worker.opts[:handler] || Sneakers::CONFIG[:handler]
+    handler = handler_klass.new(@channel, queue, worker.opts)
+
+    @consumer = queue.subscribe(:block => false, :ack => @opts[:ack]) do | delivery_info, metadata, msg |
+      worker.do_work(delivery_info, metadata, msg, handler)
     end
     nil
   end

data/lib/sneakers/runner.rb

@@ -41,7 +41,7 @@ module Sneakers
 
     def reload_config!
       Sneakers.logger.warn("Loading runner configuration...")
-      config_file = Sneakers::Config[:runner_config_file]
+      config_file = Sneakers::CONFIG[:runner_config_file]
 
       if config_file
         begin
@@ -55,7 +55,7 @@ module Sneakers
       config = make_serverengine_config
 
       [:before_fork, :after_fork].each do | hook |
-        Sneakers::Config[:hooks][hook] = config.delete(hook) if config[hook]
+        Sneakers::CONFIG[:hooks][hook] = config.delete(hook) if config[hook]
       end
 
 
@@ -65,11 +65,11 @@ module Sneakers
 
   private
     def make_serverengine_config
-      Sneakers::Config.merge(@conf).merge({
+      Sneakers::CONFIG.merge(@conf).merge({
         :worker_type => 'process',
         :worker_classes => @worker_classes
       })
     end
   end
-  
+
 end

data/lib/sneakers/version.rb

@@ -1,3 +1,3 @@
 module Sneakers
-  VERSION = "0.1.1.pre"
+  VERSION = "1.0.0"
 end