sneakers_handlers 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a3e5a73cc16eed57ed53952d2989b5d9dc18e47
-  data.tar.gz: ba629ecd84e6b69f8477746bbb57dd360c2cc06b
+  metadata.gz: f8282f82b86f3812de85b874ad7f000ff38cc382
+  data.tar.gz: eac3f28db8e8f77678d48e621d446bc986615216
 SHA512:
-  metadata.gz: c81a40929c55500b25f811f62f1620c8b5df1202d6cba891896ec3225f8d6f5bcdfe557b55b0375cf71c9c5fda2a901fde613690f4a61b5ab1567d43530ebe5f
-  data.tar.gz: d483161d7bc28c32ab53e34ca349b5f80d955ae869907877ebe15be10d7539317ce7865b0cc0c18ceaf8f7e81fe39d15d50fd1132031429b618f37921a1638a7
+  metadata.gz: 8618303e16ee9775d7c8e60e71928e55cfad85a7840f53bd6d912aaa8ca5f148bf011a2148ed22412ee3458a21b8cf8b89e6f96b8590f2e613d977f8e90ef88c
+  data.tar.gz: 08408381176a5f5e420aa2b3115ba6ab6559e89361c84c27423e2c322699c9a29dfc535affc9a5f67ccfffb6f9047b60b2d2c063961f91f68471641cc2665af6

data/README.md

@@ -22,7 +22,7 @@ Or install it yourself as:
 
 ## Usage
 
-The gem introduces two handlers you can use as part of your sneaker workers: `SneakersHandlers::DeadLetter` and `SneakersHandlers::RetryHandler`
+The gem introduces two handlers you can use as part of your sneaker workers: `SneakersHandlers::DeadLetterHandler` and `SneakersHandlers::RetryHandler`
 
 ## Using the `SneakersHandlers::RetryHandler` handler
 
@@ -49,7 +49,34 @@ class MyWorker
   def work(payload)
     ...
   end
-end                 
+end
+```
+
+## Using the `SneakersHandlers::ExponentialBackoffHandler` handler
+
+Exponential Backoff isn't really the right phrase for this. It's more
+static configurable backoff. Plan on updating the name in the future.
+
+When defining your worker, you have the following extra options:
+
+`delay` [required] : An array containing the number of seconds to pause between retries
+
+```ruby
+class MyWorker
+  include Sneakers::Worker
+  from_queue "my-app.resource_processor",
+      durable: true,
+      ack: true,
+      exchange: "domain_events",
+      exchange_type: :topic,
+      routing_key: "resources.lifecycle.*",
+      handler: SneakersHandlers::ExponentialBackoffHandler,
+      delay: [1.second, 10.seconds, 1.minute, 10.minutes]
+
+  def work(payload)
+    ...
+  end
+end
 ```
 
 ## Development

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
-task :default => :spec
+task :default => :test

data/bin/rake

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/lib/sneakers_handlers.rb

@@ -1,7 +1,8 @@
 require "sneakers_handlers/version"
-require 'sneakers'
-require 'sneakers_handlers/dead_letter'
-require 'sneakers_handlers/retry_handler'
+require "sneakers"
+require "sneakers_handlers/dead_letter_handler"
+require "sneakers_handlers/retry_handler"
+require "sneakers_handlers/exponential_backoff_handler"
+
 module SneakersHandlers
-  # Your code goes here...
 end

data/lib/sneakers_handlers/{dead_letter.rb → dead_letter_handler.rb}

@@ -1,5 +1,5 @@
 module SneakersHandlers
-  class DeadLetter < ::Sneakers::Handlers::Oneshot
+  class DeadLetterHandler < ::Sneakers::Handlers::Oneshot
     def initialize(channel, queue, options)
       super
 

data/lib/sneakers_handlers/exponential_backoff_handler.rb

@@ -0,0 +1,146 @@
+# Using this handler, failed messages will be retried with an exponential
+# backoff delay, for a certain number of times, until they are dead-lettered.
+#
+# To use it you need to defined this handler in your worker:
+#
+# from_queue "my-app.queue_name",
+#   exchange: "my_exchange_name",
+#   routing_key: "my_routing_key",
+#   handler: SneakersHandlers::ExponentialBackoffHandler,
+#   arguments: { "x-dead-letter-exchange" => "my_exchange_name.dlx",
+#                "x-dead-letter-routing-key" => "my-app.queue_name" }}
+#
+# By default it will retry 25 times before dead-lettering a message, but you can
+# also customize that with the `max_retries` option:
+#
+# from_queue "my-app.queue_name",
+#   exchange: "my_exchange_name",
+#   routing_key: "my_routing_key",
+#   max_retries: 10,
+#   handler: SneakersHandlers::ExponentialBackoffHandler,
+#   arguments: { "x-dead-letter-exchange" => "my_exchange_name.dlx",
+#                "x-dead-letter-routing-key" => "my-app.queue_name" }}
+
+module SneakersHandlers
+  class ExponentialBackoffHandler
+    attr_reader :queue, :channel, :options, :max_retries
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    def initialize(channel, queue, options)
+      @queue = queue
+      @channel = channel
+      @options = options
+      @max_retries = options[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+
+      create_error_exchange!
+
+      queue.bind(primary_exchange, routing_key: queue.name)
+    end
+
+    def acknowledge(delivery_info, _, _)
+      channel.acknowledge(delivery_info.delivery_tag, false)
+    end
+
+    def reject(delivery_info, properties, message, _requeue = true)
+      retry_message(delivery_info, properties, message, :reject)
+    end
+
+    def error(delivery_info, properties, message, err)
+      retry_message(delivery_info, properties, message, err)
+    end
+
+    def timeout(delivery_info, properties, message)
+      retry_message(delivery_info, properties, message, :timeout)
+    end
+
+    def noop(_delivery_info, _properties, _message)
+    end
+
+    private
+
+    def retry_message(delivery_info, properties, message, reason)
+      attempt_number = death_count(properties[:headers])
+
+      if attempt_number < max_retries
+        delay = seconds_to_delay(attempt_number)
+
+        log("msg=retrying, delay=#{delay}, count=#{attempt_number}, properties=#{properties}, reason=#{reason}")
+
+        routing_key = "#{queue.name}.#{delay}"
+
+        retry_queue = create_retry_queue!(delay)
+        retry_queue.bind(primary_exchange, routing_key: routing_key)
+
+        primary_exchange.publish(message, routing_key: routing_key, headers: properties[:headers])
+        acknowledge(delivery_info, properties, message)
+      else
+        log("msg=erroring, count=#{attempt_number}, properties=#{properties}")
+        channel.reject(delivery_info.delivery_tag)
+      end
+    end
+
+    def death_count(headers)
+      return 0 if headers.nil? || headers["x-death"].nil?
+
+      headers["x-death"].inject(0) do |sum, x_death|
+        sum + x_death["count"] if x_death["queue"] =~ /^#{queue.name}/
+      end
+    end
+
+    def log(message)
+      Sneakers.logger.info do
+        "[#{self.class}] #{message}"
+      end
+    end
+
+    def create_exchange(name)
+      log("creating exchange=#{name}")
+
+      channel.exchange(name, type: "topic", durable: options[:exchange_options][:durable])
+    end
+
+    def primary_exchange
+      @primary_exchange ||= create_exchange("#{options[:exchange]}")
+    end
+
+    def create_error_exchange!
+      arguments = options[:queue_options][:arguments]
+
+      dlx_exchange_name = arguments.fetch("x-dead-letter-exchange")
+      dlx_routing_key = arguments.fetch("x-dead-letter-routing-key")
+
+      @error_exchange ||= create_exchange(dlx_exchange_name).tap do |exchange|
+        queue = channel.queue("#{@queue.name}.error", durable: options[:queue_options][:durable])
+        queue.bind(exchange, routing_key: dlx_routing_key)
+      end
+    end
+
+    def create_retry_queue!(delay)
+      clear_queues_cache
+      channel.queue( "#{queue.name}.retry.#{delay}",
+         durable: options[:queue_options][:durable],
+         arguments: {
+           :"x-dead-letter-exchange" => options[:exchange],
+           :"x-dead-letter-routing-key" => queue.name,
+           :"x-message-ttl" => delay * 1_000,
+           :"x-expires" => delay * 1_000 * 2
+         }
+        )
+    end
+
+    def seconds_to_delay(count)
+      (count + 1) ** 2
+    end
+
+    # When we create a new queue, `Bunny` stores its name in an internal cache.
+    # The problem is that as we are creating ephemeral queues that can expire shortly
+    # after they are created, this cached queue may not exist anymore when we try to
+    # publish a second message to it.
+    # Removing queues from the cache guarantees that `Bunny` will always try
+    # to check if they exist, and when they don't, it will create them for us.
+    def clear_queues_cache
+      channel.queues.clear
+    end
+  end
+end

data/lib/sneakers_handlers/version.rb

@@ -1,3 +1,3 @@
 module SneakersHandlers
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/sneakers_handlers.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "dotenv"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "minitest-reporters"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sneakers_handlers
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - John Bohn, Abe Petrillo, Brian Storti
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-12 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sneakers
@@ -67,7 +67,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,10 +107,12 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/rake
 - bin/setup
 - circle.yml
 - lib/sneakers_handlers.rb
-- lib/sneakers_handlers/dead_letter.rb
+- lib/sneakers_handlers/dead_letter_handler.rb
+- lib/sneakers_handlers/exponential_backoff_handler.rb
 - lib/sneakers_handlers/retry_handler.rb
 - lib/sneakers_handlers/version.rb
 - sneakers_handlers.gemspec
@@ -125,4 +141,3 @@ signing_key:
 specification_version: 4
 summary: Adds Handlers to use with Sneakers
 test_files: []
-has_rdoc: