resque-retry 1.5.0 → 1.7.6

data/lib/resque/failure/multiple_with_retry_suppression.rb

@@ -3,10 +3,9 @@ require 'resque/plugins/retry/logging'
 
 module Resque
   module Failure
-
     # A multiple failure backend, with retry suppression
     #
-    # For example: if you had a job that could retry 5 times, your failure 
+    # For example: if you had a job that could retry 5 times, your failure
     # backends are not notified unless the _final_ retry attempt also fails.
     #
     # Example:
@@ -28,17 +27,44 @@ module Resque
       #
       # @api private
       def save
-        log_message 'failure backend save', args_from(payload), exception
+        args = args_from(payload)
+
+        log_message 'failure backend save', args, exception
 
         retryable = retryable?
         job_being_retried = retryable && retrying?
 
         if !job_being_retried
-          log_message "#{retryable ? '' : 'non-'}retriable job is not being retried - sending failure to superclass", args_from(payload), exception
+          log_message(
+            "#{retryable ? '' : 'non-'}retryable job is not being retried - sending failure to superclass",
+            args,
+            exception
+          )
+
           cleanup_retry_failure_log!
-          super
-        elsif retry_delay > 0
-          log_message "retry_delay: #{retry_delay} > 0 - saving details in Redis", args_from(payload), exception
+          return super
+        end
+
+        # some plugins define retry_delay and have it take no arguments, so rather than break those,
+        # we'll just check here to see whether it takes the additional exception class argument or not
+        # we also allow all job args to be passed to a custom `retry_delay` method
+        retry_delay_arity = klass.method(:retry_delay).arity
+
+        calculated_retry_delay = if [-2, 2].include?(retry_delay_arity)
+          klass.retry_delay(exception.class, *args)
+        elsif [-1, 1].include?(retry_delay_arity)
+          klass.retry_delay(exception.class)
+        else
+          klass.retry_delay
+        end
+
+        if calculated_retry_delay > 0
+          log_message(
+            "retry_delay: #{calculated_retry_delay} > 0 - saving details in Redis",
+            args,
+            exception
+          )
+
           data = {
             :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
             :payload   => payload,
@@ -48,10 +74,19 @@ module Resque
             :worker    => worker.to_s,
             :queue     => queue
           }
+          data = Resque.encode(data)
 
-          Resque.redis.setex(failure_key, 2*retry_delay, Resque.encode(data))
+          Resque.redis.setex(
+            failure_key,
+            2 * calculated_retry_delay,
+            data
+          )
         else
-          log_message "retry_delay: #{retry_delay} <= 0 - ignoring", args_from(payload), exception
+          log_message(
+            "retry_delay: #{calculated_retry_delay} <= 0 - ignoring",
+            args,
+            exception
+          )
         end
       end
 
@@ -62,25 +97,45 @@ module Resque
         'failure-' + retry_key
       end
 
-      protected
-
-      # Return the class/module of the failed job.
-      def klass
-        Resque::Job.new(nil, nil).constantize(payload['class'])
+      # Monkey-patch this in for now since it is a hard requirement for the
+      # "retry-all" functionality to work via the Web UI.
+      #
+      # This can be removed when the following PR has been merged into `Resque`
+      # itself:
+      #
+      # https://github.com/resque/resque/pull/1659
+      #
+      # @api public
+      class_eval do |klass|
+        if !klass.respond_to?(:requeue_queue)
+          def klass.requeue_queue(queue)
+            classes.first.requeue_queue(queue)
+          end
+        end
       end
 
-      def retry_delay
-        klass.retry_delay
+      protected
+
+      def args_from(payload)
+        (payload || {})['args'].dup
       end
 
-      def retry_key
-        klass.redis_retry_key(*payload['args'])
+      def cleanup_retry_failure_log!
+        Resque.redis.del(failure_key) if retryable?
       end
 
       def failure_key
         self.class.failure_key(retry_key)
       end
 
+      def klass
+        Resque::Job.new(nil, nil).constantize(payload['class'])
+      end
+
+      def retry_key
+        klass.redis_retry_key(*payload['args'])
+      end
+
       def retryable?
         klass.respond_to?(:redis_retry_key)
       rescue NameError
@@ -88,15 +143,17 @@ module Resque
       end
 
       def retrying?
-        Resque.redis.exists(retry_key)
+        redis_key_exists?(retry_key)
       end
 
-      def cleanup_retry_failure_log!
-        Resque.redis.del(failure_key) if retryable?
-      end
+      private
 
-      def args_from(payload)
-        (payload || {})['args']
+      def redis_key_exists?(key)
+        if Resque.redis.respond_to?(:exists?)
+          Resque.redis.exists?(key)
+        else
+          ![false, 0].include?(Resque.redis.exists(key) || false)
+        end
       end
     end
   end

data/lib/resque/plugins/exponential_backoff.rb

@@ -52,12 +52,16 @@ module Resque
       #
       # @api private
       def self.extended(receiver)
+        retry_delay_multiplicand_min = DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN
         retry_delay_multiplicand_min = \
-          receiver.instance_variable_get("@retry_delay_multiplicand_min") || \
-            DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN
+          receiver.instance_variable_get(:@retry_delay_multiplicand_min) \
+            if receiver.instance_variable_defined?(:@retry_delay_multiplicand_min)
+
+        retry_delay_multiplicand_max = DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX
         retry_delay_multiplicand_max = \
-          receiver.instance_variable_get("@retry_delay_multiplicand_max") || \
-            DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX
+          receiver.instance_variable_get(:@retry_delay_multiplicand_max) \
+            if receiver.instance_variable_defined?(:@retry_delay_multiplicand_max)
+
         if retry_delay_multiplicand_min > retry_delay_multiplicand_max
           raise InvalidRetryDelayMultiplicandConfigurationException.new(
             %{"@retry_delay_multiplicand_min" must be less than or equal to "@retry_delay_multiplicand_max"}
@@ -76,10 +80,11 @@ module Resque
 
       # Selects the delay from the backoff strategy
       #
+      # @param _ [Exception] unused exception argument for signature parity
       # @return [Number] seconds to delay until the next retry.
       #
       # @api private
-      def retry_delay
+      def retry_delay(_ = nil)
         delay = backoff_strategy[retry_attempt] || backoff_strategy.last
         # if the values are the same don't bother generating a random number, if
         # the delta is zero, some platforms will raise an error

data/lib/resque/plugins/retry.rb

@@ -43,11 +43,24 @@ module Resque
       # @api public
       class AmbiguousRetryStrategyException < StandardError; end
 
+      # Raised if there is a problem with the configuration of resque-retry.
+      #
+      # @api public
+      class RetryConfigurationException < StandardError; end
+
       # Fail fast, when extended, if the "receiver" is misconfigured
       #
       # @api private
       def self.extended(receiver)
-        if receiver.instance_variable_get('@fatal_exceptions') && receiver.instance_variable_get('@retry_exceptions')
+        retry_exceptions = nil
+        retry_exceptions = receiver.instance_variable_get(:@retry_exceptions) \
+          if receiver.instance_variable_defined?(:@retry_exceptions)
+
+        fatal_exceptions = nil
+        fatal_exceptions = receiver.instance_variable_get(:@fatal_exceptions) \
+          if receiver.instance_variable_defined?(:@fatal_exceptions)
+
+        if fatal_exceptions && retry_exceptions
           raise AmbiguousRetryStrategyException.new(%{You can't define both "@fatal_exceptions" and "@retry_exceptions"})
         end
       end
@@ -94,11 +107,27 @@ module Resque
       # A retry limit of 0 will *never* retry.
       # A retry limit of -1 or below will retry forever.
       #
+      # The default value is: `1` or in the case of where `@retry_exceptions` is
+      # specified, and it contains one or more `Array` values, the maximum
+      # length will be used (e.g. `@retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }`
+      # would return `2` because `SystemCallError` _should_ be attempted at
+      # least twice to respect the specified configuration).
+      #
       # @return [Fixnum]
       #
       # @api public
       def retry_limit
-        @retry_limit ||= 1
+        @retry_limit ||= begin
+          default_retry_limit = 1
+          if instance_variable_defined?(:@retry_exceptions) && @retry_exceptions.is_a?(Hash)
+            @retry_exceptions.values.each do |value|
+              if value.is_a?(Array) && value.length > default_retry_limit
+                default_retry_limit = value.length
+              end
+            end
+          end
+          default_retry_limit
+        end
       end
 
       # Number of retry attempts used to try and perform the job
@@ -115,13 +144,22 @@ module Resque
 
       # @abstract
       # Number of seconds to delay until the job is retried
+      # If @retry_exceptions is a Hash and there is no delay defined for exception_class,
+      # looks for closest superclass and assigns it's delay to @retry_exceptions[exception_class]
       #
       # @return [Number] number of seconds to delay
       #
       # @api public
       def retry_delay(exception_class = nil)
-        if @retry_exceptions.is_a?(Hash)
-          delay = @retry_exceptions[exception_class] || 0
+        if \
+          !exception_class.nil? && \
+          instance_variable_defined?(:@retry_exceptions) && \
+          @retry_exceptions.is_a?(Hash)
+          delay = @retry_exceptions[exception_class] ||= begin
+            relevant_definitions = \
+              @retry_exceptions.select { |ex| exception_class <= ex }
+            relevant_definitions.any? ? relevant_definitions.sort.first[1] : 0
+          end
           # allow an array of delays.
           delay.is_a?(Array) ? delay[retry_attempt] || delay.last : delay
         else
@@ -131,7 +169,7 @@ module Resque
 
       # @abstract
       # Number of seconds to sleep after job is requeued
-      # 
+      #
       # @return [Number] number of seconds to sleep
       #
       # @api public
@@ -150,6 +188,16 @@ module Resque
         @retry_job_delegate ||= nil
       end
 
+      # @abstract
+      # Specify the queue that the job should be placed in upon failure
+      #
+      # @return [Symbol] Symbol representing queue that job should be placed in
+      #
+      # @api public
+      def retry_queue(exception, *args)
+        nil
+      end
+
       # @abstract
       # Modify the arguments used to retry the job. Use this to do something
       # other than try the exact same job again
@@ -158,15 +206,7 @@ module Resque
       #
       # @api public
       def retry_args(*args)
-        # Here for backwards compatibility. If an "args_for_retry" method exists
-        # invoke it, but warn that it is deprecated (and will be removed in a
-        # future revision)
-        if respond_to?(:args_for_retry)
-          warn "`Resque::Plugins::Retry#args_for_retry` is deprecated, please use `Resque::Plugins::Retry#retry_args` instead."
-          args_for_retry(*args)
-        else
-          args
-        end
+        args.dup
       end
 
       # @abstract
@@ -238,7 +278,7 @@ module Resque
       #
       # @api public
       def retry_exceptions
-        if @retry_exceptions.is_a?(Hash)
+        if instance_variable_defined?(:@retry_exceptions) && @retry_exceptions.is_a?(Hash)
           @retry_exceptions.keys
         else
           @retry_exceptions ||= nil
@@ -264,7 +304,7 @@ module Resque
         # if the retry limit was reached, dont bother checking anything else.
         if retry_limit_reached?
           log_message 'retry limit reached', args, exception
-          return false 
+          return false
         end
 
         # We always want to retry if the exception matches.
@@ -362,14 +402,27 @@ module Resque
 
         # some plugins define retry_delay and have it take no arguments, so rather than break those,
         # we'll just check here to see whether it takes the additional exception class argument or not
-        temp_retry_delay = ([-1, 1].include?(method(:retry_delay).arity) ? retry_delay(exception.class) : retry_delay)
+        # we also allow all job args to be passed to a custom `retry_delay` method
+        retry_delay_arity = method(:retry_delay).arity
+
+        temp_retry_delay = if [-2, 2].include?(retry_delay_arity)
+          retry_delay(exception.class, *args)
+        elsif [-1, 1].include?(retry_delay_arity)
+          retry_delay(exception.class)
+        else
+          retry_delay
+        end
+
+        retry_job_class = retry_job_delegate ? retry_job_delegate : self
 
-        retry_in_queue = retry_job_delegate ? retry_job_delegate : self
-        log_message "retry delay: #{temp_retry_delay} for class: #{retry_in_queue}", args, exception
+        retry_in_queue = retry_queue(exception, *args)
+        retry_in_queue ||= Resque.queue_from_class(retry_job_class)
+
+        log_message "retry delay: #{temp_retry_delay} for queue: #{retry_in_queue}", args, exception
 
         # remember that this job is now being retried. before_perform_retry will increment
         # this so it represents the retry count, and MultipleWithRetrySuppression uses
-        # the existence of this to determine if the job should be sent to the 
+        # the existence of this to determine if the job should be sent to the
         # parent failure backend (e.g. failed queue) or not.  Removing this means
         # jobs that fail before ::perform will be both retried and sent to the failed queue.
         Resque.redis.setnx(redis_retry_key(*args), -1)
@@ -378,9 +431,9 @@ module Resque
 
         if temp_retry_delay <= 0
           # If the delay is 0, no point passing it through the scheduler
-          Resque.enqueue(retry_in_queue, *retry_args)
+          Resque.enqueue_to(retry_in_queue, retry_job_class, *retry_args)
         else
-          Resque.enqueue_in(temp_retry_delay, retry_in_queue, *retry_args)
+          Resque.enqueue_in_with_queue(retry_in_queue, temp_retry_delay, retry_job_class, *retry_args)
         end
 
         # remove retry key from redis if we handed retry off to another queue.
@@ -406,6 +459,7 @@ module Resque
       #
       # @api private
       def before_perform_retry(*args)
+        return if Resque.inline?
         log_message 'before_perform_retry', args
         @on_failure_retry_hook_already_called = false
 
@@ -418,7 +472,8 @@ module Resque
         # set/update the "retry_key" expiration
         if expire_retry_key_after
           log_message "updating expiration for retry key: #{retry_key}", args
-          Resque.redis.expire(retry_key, retry_delay + expire_retry_key_after)
+          exception_class = Object.const_get(args[0]) rescue nil
+          Resque.redis.expire(retry_key, retry_delay(exception_class) + expire_retry_key_after)
         end
       end
 
@@ -428,6 +483,7 @@ module Resque
       #
       # @api private
       def after_perform_retry(*args)
+        return if Resque.inline?
         log_message 'after_perform_retry, clearing retry key', args
         clean_retry_key(*args)
       end
@@ -443,16 +499,25 @@ module Resque
       #
       # @api private
       def on_failure_retry(exception, *args)
+        return if Resque.inline?
         log_message 'on_failure_retry', args, exception
         if exception.is_a?(Resque::DirtyExit)
           # This hook is called from a worker processes, not the job process
           # that failed with a DirtyExit, so @retry_attempt wasn't set yet
           @retry_attempt = Resque.redis.get(redis_retry_key(*args)).to_i
-        elsif @on_failure_retry_hook_already_called
+        elsif instance_variable_defined?(:@on_failure_retry_hook_already_called) && \
+          @on_failure_retry_hook_already_called
           log_message 'on_failure_retry_hook_already_called', args, exception
           return
         end
 
+        # If we are "ignoring" the exception, then we decrement the retry
+        # counter, so that the current attempt didn't count toward the retry
+        # counter.
+        if ignore_exceptions.include?(exception.class)
+          @retry_attempt = Resque.redis.decr(redis_retry_key(*args))
+        end
+
         if retry_criteria_valid?(exception, *args)
           try_again(exception, *args)
         else
@@ -584,6 +649,10 @@ module Resque
         end
       end
 
+      def ignore_exceptions
+        @ignore_exceptions ||= []
+      end
+
       # Helper to call functions that may be passed as Symbols or Procs. If
       # a symbol, it is assumed to refer to a method that is already defined
       # on this class.

data/lib/resque-retry/server.rb

@@ -66,15 +66,23 @@ module ResqueRetry
       # cancels job retry
       def cancel_retry(job)
         klass = get_class(job)
-        retry_key = retry_key_for_job(job)
-        Resque.remove_delayed(klass, *job['args'])
-        Resque.redis.del("failure-#{retry_key}")
-        Resque.redis.del(retry_key)
+        if klass
+          retry_key = retry_key_for_job(job)
+          Resque.remove_delayed(klass, *job['args'])
+          Resque.redis.del("failure-#{retry_key}")
+          Resque.redis.del(retry_key)
+        else
+          raise 'cannot cancel, job not found'
+        end
       end
 
       private
       def get_class(job)
-        Resque::Job.new(nil, nil).constantize(job['class'])
+        begin
+          Resque::Job.new(nil, nil).constantize(job['class'])
+        rescue
+          nil
+        end
       end
     end
 

data/lib/resque-retry/version.rb

@@ -1,3 +1,3 @@
 module ResqueRetry
-  VERSION = '1.5.0'
+  VERSION = '1.7.6'
 end

data/resque-retry.gemspec

@@ -25,11 +25,10 @@ Gem::Specification.new do |s|
   s.homepage = 'http://github.com/lantins/resque-retry'
   s.license = 'MIT'
 
-  s.has_rdoc = false
   s.files = `git ls-files`.split($/)
   s.require_paths = %w[lib]
 
-  s.add_dependency('resque', '~> 1.25')
+  s.add_dependency('resque', '>= 1.25', '< 3.0')
   s.add_dependency('resque-scheduler', '~> 4.0')
 
   s.add_development_dependency('rake', '~> 10.3')

data/test/exponential_backoff_test.rb

@@ -128,4 +128,11 @@ class ExponentialBackoffTest < Minitest::Test
     assert_equal 4, Resque.info[:failed],     'failed jobs'
     assert_equal 0, Resque.info[:pending],    'pending jobs'
   end
+
+  def test_backoff_with_expiration
+    Resque.redis.expects(:expire)
+
+    Resque.enqueue(ExponentialBackoffWithExpiryJob)
+    perform_next_job(@worker)
+  end
 end

data/test/ignore_exceptions_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class IgnoreExceptionsTest < Minitest::Test
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_ignore_exceptions
+    Resque.enqueue(IgnoreExceptionsJob)
+    retry_key = IgnoreExceptionsJob.redis_retry_key
+
+    IgnoreExceptionsJob.stubs(:perform).raises(AnotherCustomException)
+    perform_next_job(@worker)
+    assert_equal '0', Resque.redis.get(retry_key), 'retry counter'
+
+    IgnoreExceptionsJob.stubs(:perform).raises(AnotherCustomException)
+    perform_next_job(@worker)
+    assert_equal '1', Resque.redis.get(retry_key), 'retry counter'
+
+    IgnoreExceptionsJob.stubs(:perform).raises(CustomException)
+    perform_next_job(@worker)
+    assert_equal '1', Resque.redis.get(retry_key), 'retry counter'
+  end
+end

data/test/multiple_failure_test.rb

@@ -1,4 +1,4 @@
-require 'test_helper'
+require_relative './test_helper'
 
 # Mock failure backend for testing MultipleWithRetrySuppression
 class MockFailureBackend < Resque::Failure::Base
@@ -14,7 +14,6 @@ class MockFailureBackend < Resque::Failure::Base
 end
 
 class MultipleFailureTest < Minitest::Test
-
   def setup
     Resque.redis.flushall
     @worker = Resque::Worker.new(:testing)
@@ -26,9 +25,8 @@ class MultipleFailureTest < Minitest::Test
     Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
   end
 
-  def failure_key_for(klass)
-    args = []
-    key = 'failure-' + klass.redis_retry_key(args)
+  def failure_key_for(klass, *args)
+    Resque::Failure::MultipleWithRetrySuppression.failure_key(klass.redis_retry_key(args))
   end
 
   def test_failure_is_passed_on_when_job_class_not_found
@@ -41,7 +39,9 @@ class MultipleFailureTest < Minitest::Test
     perform_next_job(@worker)
 
     assert_equal 1, MockFailureBackend.errors.count, 'should have one error'
-    assert_match /uninitialized constant.* LimitThreeJobTemp/, MockFailureBackend.errors.first
+
+    uninitialized_constant_pattern = /uninitialized constant.* LimitThreeJobTemp/
+    assert_match uninitialized_constant_pattern, MockFailureBackend.errors.first
   end
 
   def test_last_failure_is_saved_in_redis_if_delay
@@ -53,6 +53,22 @@ class MultipleFailureTest < Minitest::Test
     assert Resque.redis.exists(key)
   end
 
+  def test_retry_delay_is_calculated_with_custom_calculation
+    delay = 5
+    Resque.enqueue(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
+    perform_next_job(@worker)
+
+    key = failure_key_for(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
+    ttl = Resque.redis.ttl(key)
+    assert Resque.redis.exists(key)
+    assert MockFailureBackend.errors.size == 0
+
+    # expiration on failure_key is set to 2x the delay
+    # to ensure the customized delay is properly calculated using
+    # dynamic retry_delay method on the job
+    assert ttl > delay && delay <= (delay * 2)
+  end
+
   def test_retry_key_splatting_args
     # were expecting this to be called three times:
     # - once when we queue the job to try again
@@ -141,6 +157,20 @@ class MultipleFailureTest < Minitest::Test
     end
   end
 
+  def test_redis_exists_returns_integer
+    Resque.enqueue(RetryDefaultsJob)
+    original = Redis.exists_returns_integer
+    Redis.exists_returns_integer = true
+
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    Redis.exists_returns_integer = original
+
+    assert_equal 1, MockFailureBackend.errors.size
+  end
+
   def teardown
     Resque::Failure.backend = @old_failure_backend
   end

data/test/resque_inline_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class ResqueInlineTest < Minitest::Test
+  def setup
+    Resque.inline = true
+    Resque.expects(:redis).never
+  end
+
+  def teardown
+    Resque.inline = false
+  end
+
+  def test_runs_inline
+    GoodJob.expects :perform
+    Resque.enqueue(GoodJob)
+  end
+
+  def test_fails_inline
+    assert_raises CustomException do
+      Resque.enqueue(RetryCustomExceptionsJob, 'CustomException')
+    end
+  end
+end

data/test/resque_test.rb

@@ -1,11 +1,15 @@
 require 'test_helper'
 
-# make sure the worlds not fallen from beneith us.
 class ResqueTest < Minitest::Test
   def test_resque_version
-    major, minor, patch = Resque::Version.split('.')
-    assert_equal 1, major.to_i, 'major version does not match'
-    assert_operator minor.to_i, :>=, 8, 'minor version is too low'
+    major, minor, _ = Resque::Version.split('.')
+    assert [1, 2].include?(major.to_i), 'major version does not match'
+
+    if major.to_i == 1
+      assert_operator minor.to_i, :>=, 25, 'minor version is too low'
+    else
+      assert_operator minor.to_i, :>=, 0, 'minor version is too low'
+    end
   end
 
   def test_good_job

data/test/retry_custom_delay_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class RetryCustomDelayTest < Minitest::Test
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_delay_with_exception
+    Resque.enqueue(DynamicDelayedJobOnException, 'arg1')
+    Resque.expects(:enqueue_in_with_queue).with(:testing, 4, DynamicDelayedJobOnException, 'arg1')
+
+    perform_next_job(@worker)
+  end
+
+  def test_delay_with_exception_and_args
+    Resque.enqueue(DynamicDelayedJobOnExceptionAndArgs, '3')
+    Resque.expects(:enqueue_in_with_queue).with(:testing, 3, DynamicDelayedJobOnExceptionAndArgs, '3')
+
+    perform_next_job(@worker)
+  end
+end