resque-retry 1.5.2 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d298842f059d9cd8ee69821e345e60d8a792e68f8f7b9fb2041508cd57e540c3
-  data.tar.gz: 203fc1782c1bdcf08858075f110e34421d1e5b805869514a36b3122739f2487f
+  metadata.gz: 6aa034bb302b1946fcc3900ef412f35ab3d56aa18718304b6ec91ed997020d29
+  data.tar.gz: ead7a803c822ec57b55994621f13fa8a788b17743e1f682eb72b361b0f59eadc
 SHA512:
-  metadata.gz: 69bf34cb35ac34bf9975f4f2798dcb78dec39cfd0d373add776026aeaf5dbb594b70bfe2c0f8352701138a61095274413c3087dc14e251b96b117eea57c56207
-  data.tar.gz: b6602597785e8d13e1bb33883e852aece52d4b1e7ffc14f82843e42042c4219bfaa5cfeeec7a9a94176652b3bc59664535c218860127e7591ccd2f113ee80f83
+  metadata.gz: a71f62fe40bd8cda477a7a17071e738b16425b8b97f7122dec69eb28808167ad82ac148b58d25824b0dc554377e490c4ca7aa92df652913fb9504bf1a69054ef
+  data.tar.gz: f087615ff9a87816f9bda22ac9258f70aad0d564e54afe9342796ca5f59b0cee40c8b8be6cc905617987e58682d33496465bf611d6caf8a30d38ab52c322cff5

data/HISTORY.md

@@ -1,3 +1,7 @@
+# 1.5.3 (2018-11-26)
+
+* Monkey patch in the `requeue_queue` method onto the `Resque::Failure` backend (this _should_ be a temporary fix)
+
 # 1.5.2 (2018-07-16)
 
 * Remove build support for `ruby < 2.3`, `jruby < 9.1` and `rbx`

data/README.md

@@ -224,7 +224,48 @@ end
 The above modification will allow your job to retry up to 10 times, with a delay
 of 120 seconds, or 2 minutes between retry attempts.
 
-You can override the `retry_delay` method to set the delay value dynamically.
+You can override the `retry_delay` method to set the delay value dynamically. For example:
+
+```ruby
+class ExampleJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  def self.retry_delay(exception)
+    if exception == SocketError
+      10
+    else
+      1
+    end
+  end
+
+  def self.perform(*args)
+    heavy_lifting
+  end
+end
+```
+
+Or, if you'd like the delay to be dependent on job arguments:
+
+```ruby
+class ExampleJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  def self.retry_delay(exception, *args)
+    # the delay is dependent on the arguments passed to the job
+    # in this case, "3" is passed as the arg and that is used as the delay
+    # make sure this method returns a integer
+    args.first.to_i
+  end
+
+  def self.perform(*args)
+    heavy_lifting
+  end
+end
+
+Resque.enqueue(ExampleJob, '3')
+```
 
 ### <a name="sleep"></a> Sleep After Requeuing
 
@@ -457,6 +498,47 @@ class DeliverViaSMSC
   end
 end
 ```
+
+### Custom Retry Queues
+
+By default, when a job is retried, it is added to the `@queue` specified in the worker. However, you may want to push the job into another (lower or higher priority) queue when the job fails. You can do this by dynamically specifying the retry queue. For example:
+
+```ruby
+class ExampleJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+  @retry_delay = 1
+
+  def self.work(*args)
+    user_id, user_mode, record_id = *args
+    
+    Resque.enqueue_to(
+      target_queue_for_args(user_id, user_mode, record_id),
+      self,
+      *args
+     )
+  end
+
+  def self.retry_queue(exception, *args)
+    target_queue_for_args(*args)
+  end
+
+  def self.perform(*args)
+    heavy_lifting
+  end
+
+  def self.target_queue_for_args(*args)
+    user_id, user_mode, record_id = *args
+
+    if user_mode
+      'high
+    else
+      'low'
+    end
+  end
+end
+```
+
 ### <a name="retry_key"></a> Job Retry Identifier/Key
 
 The retry attempt is incremented and stored in a Redis key. The key is built
@@ -594,6 +676,18 @@ Reminder: `@ignore_exceptions` should be a subset of `@retry_exceptions`.
 The inner-workings of the plugin are output to the Resque [Logger](https://github.com/resque/resque/wiki/Logging)
 when `Resque.logger.level` is set to `Logger::DEBUG`.
 
+Add `VVERBOSE=true` as an environment variable to easily set the log level to debug.
+
+### Testing
+
+To run a specific test and inspect logging output
+
+```
+bundle exec rake TEST=the_test_file.rb VVERBOSE=true
+```
+
+There are many example jobs implementing various use-cases for this gem in `test_jobs.rb`
+
 Contributing/Pull Requests
 --------------------------
 

data/lib/resque-retry/version.rb

@@ -1,3 +1,3 @@
 module ResqueRetry
-  VERSION = '1.5.2'
+  VERSION = '1.5.3'
 end

data/lib/resque/failure/multiple_with_retry_suppression.rb

@@ -3,10 +3,9 @@ require 'resque/plugins/retry/logging'
 
 module Resque
   module Failure
-
     # A multiple failure backend, with retry suppression
     #
-    # For example: if you had a job that could retry 5 times, your failure 
+    # For example: if you had a job that could retry 5 times, your failure
     # backends are not notified unless the _final_ retry attempt also fails.
     #
     # Example:
@@ -34,11 +33,21 @@ module Resque
         job_being_retried = retryable && retrying?
 
         if !job_being_retried
-          log_message "#{retryable ? '' : 'non-'}retriable job is not being retried - sending failure to superclass", args_from(payload), exception
+          log_message(
+            "#{retryable ? '' : 'non-'}retryable job is not being retried - sending failure to superclass",
+            args_from(payload),
+            exception
+          )
+
           cleanup_retry_failure_log!
           super
         elsif retry_delay > 0
-          log_message "retry_delay: #{retry_delay} > 0 - saving details in Redis", args_from(payload), exception
+          log_message(
+            "retry_delay: #{retry_delay} > 0 - saving details in Redis",
+            args_from(payload),
+            exception
+          )
+
           data = {
             :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
             :payload   => payload,
@@ -48,10 +57,19 @@ module Resque
             :worker    => worker.to_s,
             :queue     => queue
           }
+          data = Resque.encode(data)
 
-          Resque.redis.setex(failure_key, 2*retry_delay, Resque.encode(data))
+          Resque.redis.setex(
+            failure_key,
+            2 * retry_delay,
+            data
+          )
         else
-          log_message "retry_delay: #{retry_delay} <= 0 - ignoring", args_from(payload), exception
+          log_message(
+            "retry_delay: #{retry_delay} <= 0 - ignoring",
+            args_from(payload),
+            exception
+          )
         end
       end
 
@@ -62,9 +80,33 @@ module Resque
         'failure-' + retry_key
       end
 
+      # Monkey-patch this in for now since it is a hard requirement for the
+      # "retry-all" functionality to work via the Web UI.
+      #
+      # This can be removed when the following PR has been merged into `Resque`
+      # itself:
+      #
+      # https://github.com/resque/resque/pull/1659
+      #
+      # @api public
+      def self.requeue_queue(queue)
+        classes.first.requeue_queue(queue)
+      end
+
       protected
 
-      # Return the class/module of the failed job.
+      def args_from(payload)
+        (payload || {})['args']
+      end
+
+      def cleanup_retry_failure_log!
+        Resque.redis.del(failure_key) if retryable?
+      end
+
+      def failure_key
+        self.class.failure_key(retry_key)
+      end
+
       def klass
         Resque::Job.new(nil, nil).constantize(payload['class'])
       end
@@ -77,10 +119,6 @@ module Resque
         klass.redis_retry_key(*payload['args'])
       end
 
-      def failure_key
-        self.class.failure_key(retry_key)
-      end
-
       def retryable?
         klass.respond_to?(:redis_retry_key)
       rescue NameError
@@ -90,14 +128,6 @@ module Resque
       def retrying?
         Resque.redis.exists(retry_key)
       end
-
-      def cleanup_retry_failure_log!
-        Resque.redis.del(failure_key) if retryable?
-      end
-
-      def args_from(payload)
-        (payload || {})['args']
-      end
     end
   end
 end

data/lib/resque/plugins/retry.rb

@@ -54,23 +54,10 @@ module Resque
       def self.extended(receiver)
         retry_exceptions = receiver.instance_variable_get('@retry_exceptions')
         fatal_exceptions = receiver.instance_variable_get('@fatal_exceptions')
-        ignore_exceptions = receiver.instance_variable_get('@ignore_exceptions')
 
         if fatal_exceptions && retry_exceptions
           raise AmbiguousRetryStrategyException.new(%{You can't define both "@fatal_exceptions" and "@retry_exceptions"})
         end
-
-        # Check that ignore_exceptions is a subset of retry_exceptions
-        if retry_exceptions.is_a?(Hash)
-          exceptions = retry_exceptions.keys
-        else
-          exceptions = Array(retry_exceptions)
-        end
-
-        excess_exceptions = Array(ignore_exceptions) - exceptions
-        unless excess_exceptions.empty?
-          raise RetryConfigurationException, "The following exceptions are defined in @ignore_exceptions but not in @retry_exceptions: #{excess_exceptions.join(', ')}."
-        end
       end
 
       # Copy retry criteria checks, try again callbacks, and give up callbacks
@@ -176,6 +163,16 @@ module Resque
         @retry_job_delegate ||= nil
       end
 
+      # @abstract
+      # Specify the queue that the job should be placed in upon failure
+      #
+      # @return [Symbol] Symbol representing queue that job should be placed in
+      #
+      # @api public
+      def retry_queue(exception, *args)
+        nil
+      end
+
       # @abstract
       # Modify the arguments used to retry the job. Use this to do something
       # other than try the exact same job again
@@ -388,10 +385,23 @@ module Resque
 
         # some plugins define retry_delay and have it take no arguments, so rather than break those,
         # we'll just check here to see whether it takes the additional exception class argument or not
-        temp_retry_delay = ([-1, 1].include?(method(:retry_delay).arity) ? retry_delay(exception.class) : retry_delay)
+        # we also allow all job args to be passed to a custom `retry_delay` method
+        retry_delay_arity = method(:retry_delay).arity
+
+        temp_retry_delay = if [-2, 2].include?(retry_delay_arity)
+          retry_delay(exception.class, *args)
+        elsif [-1, 1].include?(retry_delay_arity)
+          retry_delay(exception.class)
+        else
+          retry_delay
+        end
+
+        retry_job_class = retry_job_delegate ? retry_job_delegate : self
+
+        retry_in_queue = retry_queue(exception, *args)
+        retry_in_queue ||= Resque.queue_from_class(retry_job_class)
 
-        retry_in_queue = retry_job_delegate ? retry_job_delegate : self
-        log_message "retry delay: #{temp_retry_delay} for class: #{retry_in_queue}", args, exception
+        log_message "retry delay: #{temp_retry_delay} for queue: #{retry_in_queue}", args, exception
 
         # remember that this job is now being retried. before_perform_retry will increment
         # this so it represents the retry count, and MultipleWithRetrySuppression uses
@@ -404,9 +414,9 @@ module Resque
 
         if temp_retry_delay <= 0
           # If the delay is 0, no point passing it through the scheduler
-          Resque.enqueue(retry_in_queue, *retry_args)
+          Resque.enqueue_to(retry_in_queue, retry_job_class, *retry_args)
         else
-          Resque.enqueue_in(temp_retry_delay, retry_in_queue, *retry_args)
+          Resque.enqueue_in_with_queue(retry_in_queue, temp_retry_delay, retry_job_class, *retry_args)
         end
 
         # remove retry key from redis if we handed retry off to another queue.

data/test/ignore_exceptions_test.rb

@@ -23,16 +23,4 @@ class IgnoreExceptionsTest < Minitest::Test
     perform_next_job(@worker)
     assert_equal '1', Resque.redis.get(retry_key), 'retry counter'
   end
-
-  def test_ignore_exception_configuration_1
-    assert_raises Resque::Plugins::Retry::RetryConfigurationException do
-      IgnoreExceptionsImproperlyConfiguredJob1.extend(Resque::Plugins::Retry)
-    end
-  end
-
-  def test_ignore_exception_configuration_2
-    assert_raises Resque::Plugins::Retry::RetryConfigurationException do
-      IgnoreExceptionsImproperlyConfiguredJob2.extend(Resque::Plugins::Retry)
-    end
-  end
 end

data/test/retry_custom_delay_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class RetryCustomDelayTest < Minitest::Test
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_delay_with_exception
+    Resque.enqueue(DynamicDelayedJobOnException, 'arg1')
+    Resque.expects(:enqueue_in_with_queue).with(:testing, 4, DynamicDelayedJobOnException, 'arg1')
+
+    perform_next_job(@worker)
+  end
+
+  def test_delay_with_exception_and_args
+    Resque.enqueue(DynamicDelayedJobOnExceptionAndArgs, '3')
+    Resque.expects(:enqueue_in_with_queue).with(:testing, 3, DynamicDelayedJobOnExceptionAndArgs, '3')
+
+    perform_next_job(@worker)
+  end
+end

data/test/retry_queue_test.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+
+class RetryQueueTest < Minitest::Test
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_retry_delayed_failed_jobs_in_separate_queue
+    Resque.enqueue(DelayedJobWithRetryQueue, 'arg1')
+    Resque.expects(:enqueue_in_with_queue).with(:testing_retry_delegate, 1, JobRetryQueue, 'arg1')
+
+    perform_next_job(@worker)
+  end
+
+  def test_retry_delayed_failed_jobs_in_dynamic_queue
+    queue_name = "dynamic_queue_#{Time.now.to_i}"
+
+    Resque.enqueue(JobWithDynamicRetryQueue, queue_name)
+    Resque.expects(:enqueue_in_with_queue).with(queue_name, 1, JobWithDynamicRetryQueue, queue_name)
+
+    perform_next_job(@worker)
+  end
+end

data/test/retry_test.rb

@@ -221,7 +221,7 @@ class RetryTest < Minitest::Test
 
     perform_next_job(@worker)
 
-    assert job_from_retry_queue = Resque.pop(:testing_retry)
+    assert job_from_retry_queue = Resque.pop(:testing_retry_delegate)
     assert_equal ['arg1'], job_from_retry_queue['args']
     assert_nil Resque.redis.get(JobWithRetryQueue.redis_retry_key('arg1'))
   end
@@ -234,13 +234,6 @@ class RetryTest < Minitest::Test
     perform_next_job(@worker)
   end
 
-  def test_retry_delayed_failed_jobs_in_separate_queue
-    Resque.enqueue(DelayedJobWithRetryQueue, 'arg1')
-    Resque.expects(:enqueue_in).with(1, JobRetryQueue, 'arg1')
-
-    perform_next_job(@worker)
-  end
-
   def test_delete_redis_key_when_job_is_successful
     Resque.enqueue(GoodJob, 'arg1')
 

data/test/test_helper.rb

@@ -2,13 +2,6 @@ dir = File.dirname(File.expand_path(__FILE__))
 $LOAD_PATH.unshift dir + '/../lib'
 $TESTING = true
 
-require 'rubygems'
-require 'timeout'
-require 'minitest/autorun'
-require 'minitest/pride'
-require 'rack/test'
-require 'mocha/setup'
-
 # Run code coverage in MRI 1.9 only.
 if RUBY_VERSION >= '1.9' && RUBY_ENGINE == 'ruby'
   require 'simplecov'
@@ -17,6 +10,13 @@ if RUBY_VERSION >= '1.9' && RUBY_ENGINE == 'ruby'
   end
 end
 
+require 'rubygems'
+require 'timeout'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'rack/test'
+require 'mocha/setup'
+
 require 'resque-retry'
 require dir + '/test_jobs'
 

data/test/test_jobs.rb

@@ -56,7 +56,7 @@ end
 
 class JobRetryQueue
   extend Resque::Plugins::Retry
-  @queue = :testing_retry
+  @queue = :testing_retry_delegate
 
   def self.perform(*args)
   end
@@ -72,6 +72,50 @@ class JobWithRetryQueue
   end
 end
 
+class JobWithDynamicRetryQueue
+  extend Resque::Plugins::Retry
+  @queue = :testing
+  @retry_delay = 1
+
+  def self.retry_queue(exception, *args)
+    args.first
+  end
+
+  def self.perform(*args)
+    raise
+  end
+end
+
+class DynamicDelayedJobOnExceptionAndArgs
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  def self.retry_delay(exception, *args)
+    args.first.to_i
+  end
+
+  def self.perform(*args)
+    raise
+  end
+end
+
+class DynamicDelayedJobOnException
+  extend Resque::Plugins::Retry
+  @queue = :testing
+
+  def self.retry_delay(exception)
+    if exception == SocketError
+      4
+    else
+      1
+    end
+  end
+
+  def self.perform(*args)
+    raise SocketError
+  end
+end
+
 class DelayedJobWithRetryQueue
   extend Resque::Plugins::Retry
   @queue = :testing
@@ -591,16 +635,3 @@ class IgnoreExceptionsJob
     "Hello, World!"
   end
 end
-
-class IgnoreExceptionsImproperlyConfiguredJob1
-  # Manually extend Resque::Plugins::Retry in the test code to catch the
-  # exception.
-  @ignore_exceptions = [CustomException]
-end
-
-class IgnoreExceptionsImproperlyConfiguredJob2
-  # Manually extend Resque::Plugins::Retry in the test code to catch the
-  # exception.
-  @ignore_exceptions = [CustomException]
-  @retry_exceptions = { AnotherCustomException => 0 }
-end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-retry
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.5.3
 platform: ruby
 authors:
 - Luke Antins
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-16 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -188,8 +188,10 @@ files:
 - test/resque_test.rb
 - test/retry_callbacks_test.rb
 - test/retry_criteria_test.rb
+- test/retry_custom_delay_test.rb
 - test/retry_exception_delay_test.rb
 - test/retry_inheriting_checks_test.rb
+- test/retry_queue_test.rb
 - test/retry_test.rb
 - test/server_helpers_test.rb
 - test/server_test.rb