resque-retry 1.1.1 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eaaf3cd5c50ad76cf580f632bd6597db0531ffa5
-  data.tar.gz: c621c1f3446b60d415ba4d3d696d945b5c824837
+  metadata.gz: b1b6a97e99754f141a99af11bd0d77f5f73d8010
+  data.tar.gz: 2c6e42f2294034e932b4a12998770214a0eed1ea
 SHA512:
-  metadata.gz: 594beb0db07135808c5e761fa1519c3d3c101492cf625f955dcbe8d68749c747458ce02cf782f7383292de86f2e9368a131503aa936692eb4c5e629947fc01de
-  data.tar.gz: cbdcf78bab11246287808884994446042866c2a39c8c83595258479db7db53b3565e987c1ac919565e179d768f4543dcf4d8417695a7b876023242a86673ed31
+  metadata.gz: d748027d00df3b099e92cac7123baedccca96f0240a5f17c27f15277a57c0c527a3207e277950ce42b57a3f440791cf75e2245d4a920db35f897d126f932a895
+  data.tar.gz: 56ce881aff8343f9d381861aa38a061a6bf80eaf3e099f90d6a41ca6a3ca3fef52cb30cb12a3014809400d6c52b5f1f1c0103fd8ee6454878b0f512e0f8c2daf

data/HISTORY.md

@@ -1,5 +1,11 @@
 ## HEAD
 
+## 1.1.4 (2014-03-17)
+
+* Fixed displaying retry information in resque web interface, caused by `Resque::Helpers` being deprecated.
+* Feature: Allow `@fatal_exceptions` as inverse of `@retry_exceptions`, when fatal exception is raised the job will be immediately fail.
+* Feature: Allow a random retry delay (within a range) when using exponential backoff strategy.
+
 ## 1.1.1 (2014-03-12)
 
 * Adjust gem dependency `resque-scheduler`.

data/README.md

@@ -208,6 +208,8 @@ Use this if you wish to vary the delay between retry attempts:
 
                   no delay, 1m, 10m,   1h,    3h,    6h
     @backoff_strategy = [0, 60, 600, 3600, 10800, 21600]
+    @retry_delay_multiplicand_min = 1.0
+    @retry_delay_multiplicand_max = 1.0
 
 The first delay will be 0 seconds, the 2nd will be 60 seconds, etc...
 Again, tweak to your own needs.
@@ -215,6 +217,13 @@ Again, tweak to your own needs.
 The number of retries is equal to the size of the `backoff_strategy`
 array, unless you set `retry_limit` yourself.
 
+The delay values will be multiplied by a random `Float` value between
+`retry_delay_multiplicand_min` and `retry_delay_multiplicand_max` (both have a
+default of `1.0`). The product (`delay_multiplicand`) is recalculated on every
+attempt. This feature can be useful if you have a lot of jobs fail at the same
+time (e.g. rate-limiting/throttling or connectivity issues) and you don't want
+them all retried on the same schedule.
+
 ### Retry Specific Exceptions
 
 The default will allow a retry for any type of exception. You may change
@@ -256,6 +265,26 @@ In the above example, Resque would retry any `DeliverSMS` jobs which throw a
 will be retried 30 seconds later, if it throws `SystemCallError` it will first
 retry 120 seconds later then subsequent retry attempts 240 seconds later.
 
+### Fail Fast For Specific Exceptions
+
+The default will allow a retry for any type of exception. You may change
+it so specific exceptions fail immediately by using `fatal_exceptions`:
+
+    class DeliverSMS
+      extend Resque::Plugins::Retry
+      @queue = :mt_divisions
+
+      @fatal_exceptions = [NetworkError]
+
+      def self.perform(mt_id, mobile_number, message)
+        heavy_lifting
+      end
+    end
+
+In the above example, Resque would retry any `DeliverSMS` jobs that throw any
+type of error other than `NetworkError`. If the job throws a `NetworkError` it
+will be marked as "failed" immediately.
+
 ### Custom Retry Criteria Check Callbacks
 
 You may define custom retry criteria callbacks:

data/examples/demo/Procfile

@@ -1,3 +1,4 @@
 web:       bundle exec rake start
 worker:    bundle exec rake resque:work QUEUE=*
-scheduler: bundle exec rake resque:scheduler
+scheduler: bundle exec rake resque:scheduler
+redis:     redis-server

data/examples/demo/jobs.rb

@@ -21,7 +21,7 @@ class FailingWithRetryJob
 
   @queue = :testing_failure
   @retry_limit = 4
-  @retry_delay = 3
+  @retry_delay = 20
 
   # Perform that raises an exception, but we will retry the job on failure
   def self.perform(*args)

data/lib/resque-retry/server.rb

@@ -39,14 +39,10 @@ module ResqueRetry
     module Helpers
       # builds a retry key for the specified job.
       def retry_key_for_job(job)
-        begin
-          klass = Resque.constantize(job['class'])
-          if klass.respond_to?(:redis_retry_key)
-            klass.redis_retry_key(job['args'])
-          else
-            nil
-          end
-        rescue NameError
+        klass = Resque::Job.new(nil, nil).constantize(job['class'])
+        if klass.respond_to?(:redis_retry_key)
+          klass.redis_retry_key(job['args'])
+        else
           nil
         end
       end

data/lib/resque-retry/version.rb

@@ -1,3 +1,3 @@
 module ResqueRetry
-  VERSION = '1.1.1'
+  VERSION = '1.1.4'
 end

data/lib/resque/failure/multiple_with_retry_suppression.rb

@@ -54,7 +54,7 @@ module Resque
 
       # Return the class/module of the failed job.
       def klass
-        Resque::Job.new(nil, nil).constantize payload['class']
+        Resque::Job.new(nil, nil).constantize(payload['class'])
       end
 
       def retry_delay

data/lib/resque/plugins/exponential_backoff.rb

@@ -37,6 +37,34 @@ module Resque
     module ExponentialBackoff
       include Resque::Plugins::Retry
 
+      # Raised if the min/max retry-delay multiplicand configuration is invalid
+      #
+      # @api public
+      class InvalidRetryDelayMultiplicandConfigurationException < StandardError; end
+
+      # Constants
+      #
+      # @api public
+      DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN = 1.0
+      DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX = 1.0
+
+      # Fail fast, when extended, if the "receiver" is misconfigured
+      #
+      # @api private
+      def self.extended(receiver)
+        retry_delay_multiplicand_min = \
+          receiver.instance_variable_get("@retry_delay_multiplicand_min") || \
+            DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN
+        retry_delay_multiplicand_max = \
+          receiver.instance_variable_get("@retry_delay_multiplicand_max") || \
+            DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX
+        if retry_delay_multiplicand_min > retry_delay_multiplicand_max
+          raise InvalidRetryDelayMultiplicandConfigurationException.new(
+            %{"@retry_delay_multiplicand_min" must be less than or equal to "@retry_delay_multiplicand_max"}
+          )
+        end
+      end
+
       # Defaults to the number of delays in the backoff strategy
       #
       # @return [Number] maximum number of retries
@@ -52,7 +80,32 @@ module Resque
       #
       # @api private
       def retry_delay
-        backoff_strategy[retry_attempt] || backoff_strategy.last
+        delay = backoff_strategy[retry_attempt] || backoff_strategy.last
+        delay_multiplicand = \
+          rand(retry_delay_multiplicand_min..retry_delay_multiplicand_max)
+        (delay * delay_multiplicand).to_i
+      end
+
+      # @abstract
+      # The minimum value (lower-bound) for the range that is is used in
+      # calculating the retry-delay product
+      #
+      # @return [Float]
+      #
+      # @api public
+      def retry_delay_multiplicand_min
+        @retry_delay_multiplicand_min ||= DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN
+      end
+
+      # @abstract
+      # The maximum value (upper-bound) for the range that is is used in
+      # calculating the retry-delay product
+      #
+      # @return [Float]
+      #
+      # @api public
+      def retry_delay_multiplicand_max
+        @retry_delay_multiplicand_max ||= DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX
       end
 
       # @abstract
@@ -67,4 +120,4 @@ module Resque
     end
 
   end
-end
+end

data/lib/resque/plugins/retry.rb

@@ -36,6 +36,20 @@ module Resque
     #
     module Retry
 
+      # Raised if the retry-strategy cannot be determined or has conflicts
+      #
+      # @api public
+      class AmbiguousRetryStrategyException < StandardError; end
+
+      # Fail fast, when extended, if the "receiver" is misconfigured
+      #
+      # @api private
+      def self.extended(receiver)
+        if receiver.instance_variable_get("@fatal_exceptions") && receiver.instance_variable_get("@retry_exceptions")
+          raise AmbiguousRetryStrategyException.new(%{You can't define both "@fatal_exceptions" and "@retry_exceptions"})
+        end
+      end
+
       # Copy retry criteria checks on inheritance.
       #
       # @api private
@@ -152,16 +166,45 @@ module Resque
       #
       # @api public
       def retry_exception?(exception)
-        return true if retry_exceptions.nil?
-        !! retry_exceptions.any? do |ex|
-          if exception.is_a?(Class)
-            ex >= exception
-          else
-            ex === exception
+        # If both "fatal_exceptions" and "retry_exceptions" are undefined we are
+        # done (we should retry the exception)
+        #
+        # It is intentional that we check "retry_exceptions" first since it is
+        # more likely that it will be defined (over "fatal_exceptions") as it
+        # has been part of the API for quite a while
+        return true if retry_exceptions.nil? && fatal_exceptions.nil?
+
+        # If "fatal_exceptions" is undefined interrogate "retry_exceptions"
+        if fatal_exceptions.nil?
+          retry_exceptions.any? do |ex|
+            if exception.is_a?(Class)
+              ex >= exception
+            else
+              ex === exception
+            end
+          end
+        # It is safe to assume we need to check "fatal_exceptions" at this point
+        else
+          fatal_exceptions.none? do |ex|
+            if exception.is_a?(Class)
+              ex >= exception
+            else
+              ex === exception
+            end
           end
         end
       end
 
+      # @abstract
+      # Controls what exceptions may not be retried
+      #
+      # Default: `nil` - this will retry all exceptions.
+      #
+      # @return [Array, nil]
+      #
+      # @api public
+      attr_reader :fatal_exceptions
+
       # @abstract
       # Controls what exceptions may be retried
       #

data/test/exponential_backoff_test.rb

@@ -41,6 +41,75 @@ class ExponentialBackoffTest < MiniTest::Unit::TestCase
     assert_in_delta (start_time + 21_600),  delayed[4], 1.00, '6th delay'
   end
 
+  def test_dont_allow_both_retry_and_ignore_exceptions
+    job_types = [
+      InvalidRetryDelayMaxConfigurationJob,
+      InvalidRetryDelayMinAndMaxConfigurationJob,
+      InvalidRetryDelayMinConfigurationJob,
+    ]
+
+    job_types.each do |job_type|
+      assert_raises Resque::Plugins::ExponentialBackoff::InvalidRetryDelayMultiplicandConfigurationException do
+        job_type.extend(Resque::Plugins::ExponentialBackoff)
+      end
+    end
+  end
+
+  def test_default_backoff_strategy_with_retry_delay_multiplicands
+    job_types = [
+      ExponentialBackoffWithRetryDelayMultiplicandMaxJob,
+      ExponentialBackoffWithRetryDelayMultiplicandMinJob,
+      ExponentialBackoffWithRetryDelayMultiplicandMinAndMaxJob,
+    ]
+
+    job_types.each do |job_type|
+      # all of these values are used heavily in assertions below
+      start_time = Time.now.to_i
+      multiplicand_min = job_type.public_send(:retry_delay_multiplicand_min)
+      multiplicand_max = job_type.public_send(:retry_delay_multiplicand_max)
+
+      Resque.enqueue(job_type)
+
+      # first attempt, failed but never retried
+      perform_next_job(@worker)
+      assert_equal 1, Resque.info[:pending]
+      assert_equal 1, Resque.info[:processed]
+      assert_equal 1, Resque.info[:failed]
+
+      # second attempt, first retry, should fail again
+      perform_next_job(@worker)
+      assert_equal 0, Resque.info[:pending]
+      assert_equal 2, Resque.info[:processed]
+      assert_equal 2, Resque.info[:failed]
+
+      # second delay
+      delayed = Resque.delayed_queue_peek(0, 1)
+      assert_in_delta(
+        start_time + 60 * multiplicand_min,
+        delayed[0],
+        60 * multiplicand_max
+      )
+
+      5.times do
+        Resque.enqueue(job_type)
+        perform_next_job(@worker)
+      end
+
+      # third through sixth delays
+      delayed = Resque.delayed_queue_peek(1, 5)
+      [600, 3600, 10_800, 21_600].each_with_index do |delay, index|
+        assert_in_delta(
+          start_time + delay * multiplicand_min,
+          delayed[index],
+          delay * multiplicand_max
+        )
+      end
+
+      # always reset the state before the next test case is run
+      Resque.redis.flushall
+    end
+  end
+
   def test_custom_backoff_strategy
     start_time = Time.now.to_i
     4.times do

data/test/retry_test.rb

@@ -17,6 +17,7 @@ class RetryTest < MiniTest::Unit::TestCase
   def test_default_settings
     assert_equal 1, RetryDefaultSettingsJob.retry_limit, 'default retry limit'
     assert_equal 0, RetryDefaultSettingsJob.retry_attempt, 'default number of retry attempts'
+    assert_equal nil, RetryDefaultSettingsJob.fatal_exceptions, 'default fatal exceptions; nil = none'
     assert_equal nil, RetryDefaultSettingsJob.retry_exceptions, 'default retry exceptions; nil = any'
     assert_equal 0, RetryDefaultSettingsJob.retry_delay, 'default seconds until retry'
   end
@@ -159,6 +160,36 @@ class RetryTest < MiniTest::Unit::TestCase
     assert_equal 0, Resque.info[:pending], 'pending jobs'
   end
 
+  def test_dont_allow_both_retry_and_ignore_exceptions
+    assert_raises Resque::Plugins::Retry::AmbiguousRetryStrategyException do
+      AmbiguousRetryStrategyJob.extend(Resque::Plugins::Retry)
+    end
+  end
+
+  def test_will_fail_immediately_if_exception_type_does_not_allow_retry
+    Resque.enqueue(FailOnCustomExceptionJob)
+
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 1, Resque.info[:failed], 'failed jobs'
+    assert_equal 1, Resque.info[:processed], 'processed job'
+  end
+
+  def test_will_retry_if_a_non_fatal_exception_is_raised
+    Resque.enqueue(FailOnCustomExceptionButRaiseStandardErrorJob)
+
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+  end
+
   def test_retry_failed_jobs_in_separate_queue
     Resque.enqueue(JobWithRetryQueue, 'arg1')
 

data/test/server_helpers_test.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+
+require 'resque'
+require 'resque-retry/server'
+
+class ServerHelpersTest < MiniTest::Unit::TestCase
+
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+
+    @helpers = Class.new.extend(ResqueRetry::Server::Helpers)
+  end
+
+  def test_retry_key_for_job
+    Resque.enqueue(LimitThreeJobDelay1Hour)
+    perform_next_job(@worker)
+
+    timestamp = Resque.delayed_queue_peek(0, 1).first
+    job = Resque.delayed_timestamp_peek(timestamp, 0, 1).first
+    assert_equal '0', @helpers.retry_attempts_for_job(job), 'should have 0 retry attempt'
+  end
+
+end

data/test/server_test.rb

@@ -7,6 +7,12 @@ ENV['RACK_ENV'] = 'test'
 class ServerTest < MiniTest::Unit::TestCase
   include Rack::Test::Methods
 
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
   def app
     Resque::Server
   end
@@ -22,4 +28,18 @@ class ServerTest < MiniTest::Unit::TestCase
     assert last_response.body.include?('/retry')
   end
 
+  def test_display_retry_job
+    # to begin with, we should have no retry jobs listed.
+    get '/retry'
+    assert last_response.body.include?('<b>0</b> timestamps'), 'should have 0 retry jobs'
+
+    # queue failing job that will retry.
+    Resque.enqueue(LimitThreeJobDelay1Hour)
+    perform_next_job(@worker)
+
+    # we should now have the retry job listed.
+    get '/retry'
+    assert last_response.body.include?('<b>1</b> timestamps'), 'should have 1 retry jobs'
+  end
+
 end

data/test/test_jobs.rb

@@ -140,6 +140,41 @@ class ExponentialBackoffJob < RetryDefaultsJob
   @queue = :testing
 end
 
+class ExponentialBackoffWithRetryDelayMultiplicandMinJob < RetryDefaultsJob
+  extend Resque::Plugins::ExponentialBackoff
+  @queue = :testing
+  @retry_delay_multiplicand_min = 0.5
+end
+
+class ExponentialBackoffWithRetryDelayMultiplicandMaxJob < RetryDefaultsJob
+  extend Resque::Plugins::ExponentialBackoff
+  @queue = :testing
+  @retry_delay_multiplicand_max = 3.0
+end
+
+class ExponentialBackoffWithRetryDelayMultiplicandMinAndMaxJob < RetryDefaultsJob
+  extend Resque::Plugins::ExponentialBackoff
+  @queue = :testing
+  @retry_delay_multiplicand_min = 0.5
+  @retry_delay_multiplicand_max = 3.0
+end
+
+class InvalidRetryDelayMaxConfigurationJob
+  @queue = :testing
+  @retry_delay_multiplicand_max = 0.9
+end
+
+class InvalidRetryDelayMinConfigurationJob
+  @queue = :testing
+  @retry_delay_multiplicand_min = 1.1
+end
+
+class InvalidRetryDelayMinAndMaxConfigurationJob
+  @queue = :testing
+  @retry_delay_multiplicand_min = 3.0
+  @retry_delay_multiplicand_max = 0.5
+end
+
 class CustomExponentialBackoffJob
   extend Resque::Plugins::ExponentialBackoff
   @queue = :testing
@@ -169,6 +204,35 @@ class RetryCustomExceptionsJob < RetryDefaultsJob
   end
 end
 
+class AmbiguousRetryStrategyJob
+  @queue = :testing
+
+  @fatal_exceptions = [CustomException]
+  @retry_exceptions = [AnotherCustomException]
+end
+
+class FailOnCustomExceptionJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  @fatal_exceptions = [CustomException]
+
+  def self.perform(*args)
+    raise CustomException
+  end
+end
+
+class FailOnCustomExceptionButRaiseStandardErrorJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  @fatal_exceptions = [CustomException]
+
+  def self.perform(*args)
+    raise StandardError
+  end
+end
+
 module RetryModuleDefaultsJob
   extend Resque::Plugins::Retry
   @queue = :testing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-retry
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.4
 platform: ruby
 authors:
 - Luke Antins
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-12 00:00:00.000000000 Z
+date: 2014-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -186,6 +186,7 @@ files:
 - test/retry_exception_delay_test.rb
 - test/retry_inheriting_checks_test.rb
 - test/retry_test.rb
+- test/server_helpers_test.rb
 - test/server_test.rb
 - test/test_helper.rb
 - test/test_jobs.rb