resque-retry 1.7.2 → 1.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 755ea6078469205b32e916d32d7dbb57a668e409d559b32ec7a5b21560a515c8
-  data.tar.gz: 98e9eb8627d4641771f66e8811d29bfc2e1baab52b5a38809bb4a0005e465480
+  metadata.gz: bcd42f2b0761226dd2cfa5f9c9d4ff336cda1be0f6acf972ae15cd553f88fa0b
+  data.tar.gz: d65b4261a3ff1068f9880739ea1f4647c10be957511840203331f714cb8281dc
 SHA512:
-  metadata.gz: 475ca16f0d0814aa68ca14ff1ba2ec30c037526b69432052e8b88940e26964cb6e25ce71282a8161796d91ccae1ebfff23b43956485f5ae6b820e83aa00bb06b
-  data.tar.gz: 1d6870270be5df0c1a9820879ebfef192bbaa2a7cbe16199e5c8c319bd8b5f7a5a2207105646ebc0a3a99221a1defeab520727053649cfa310025caadc7d1520
+  metadata.gz: 78c8fccf7a670f1fc0246d7e8862941eae3fb2bda8ea4d11fb1bea2a5890ed1867f50ff9389a752f3383bf5383783b32e3f63c5ef623909a28aaeba62f4576a2
+  data.tar.gz: 8e3e0ed80d154282e6dddd85b8b7b95b22949d580ad6c0c7a0fc9c3a16475dca861320f919db0f2ddf6b8dca78e8bdd846a48b86db6a5761848d6d4fcb7012a4

data/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    strategy:
+      matrix:
+        ruby-version: [2.7, 2.6]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake

data/.gitignore

@@ -14,3 +14,5 @@ tags.*
 *.gem
 .ruby-version
 .ruby-gemset
+.tags
+dump.rdb

data/HISTORY.md

@@ -1,3 +1,23 @@
+# 1.7.6 (2021-08-06)
+
+* Fix Redis 4.3 breaking saving job failures with multiple failure backend
+* Disable verbose logging by default during test-runs
+* Remove support for ruby < 2.6
+
+# 1.7.5 (2021-08-06)
+
+* Support `retry_delay` methods with different arity
+* Switch over to _GitHub_ CI for PR test-execution
+
+# 1.7.4 (2020-10-09)
+
+* Rework how the default `retry_limit` is calculated
+* Numerous documentation updates
+
+# 1.7.3 (2019-11-21)
+
+* Do not crash if job class is not found on the resque-web server
+
 # 1.7.2 (2019-11-21)
 
 * Address part 2 of issue #158

data/README.md

@@ -231,8 +231,8 @@ class ExampleJob
   extend Resque::Plugins::Retry
   @queue = :testing
 
-  def self.retry_delay(exception)
-    if exception == SocketError
+  def self.retry_delay(exception_class)
+    if exception_class == SocketError
       10
     else
       1
@@ -359,7 +359,12 @@ You may also want to specify different retry delays for different exception
 types. You may optionally set `@retry_exceptions` to a hash where the keys are
 your specific exception classes to retry on, and the values are your retry
 delays in seconds or an array of retry delays to be used similar to exponential
-backoff.
+backoff.  `resque-retry` will attempt to determine your retry strategy's
+`@retry_limit` based on your specified `@retry_exceptions`.  If, however, you
+define `@retry_limit` explicitly, you should define `@retry_limit` such that it
+allows for your retry strategies to complete.  If your `@retry_limit` is less
+than the number of desired retry attempts defined in `@retry_exceptions`, your
+job will only retry `@retry_limit` times.
 ```ruby
 class DeliverSMS
   extend Resque::Plugins::Retry
@@ -374,9 +379,15 @@ end
 ```
 
 In the above example, Resque would retry any `DeliverSMS` jobs which throw a
-`NetworkError` or `SystemCallError`. If the job throws a `NetworkError` it
-will be retried 30 seconds later, if it throws `SystemCallError` it will first
-retry 120 seconds later then subsequent retry attempts 240 seconds later.
+`NetworkError` or `SystemCallError`. The `@retry_limit` would be inferred to be
+2 based on the longest retry strategy defined in `@retry_exceptions`. If the job
+throws a `NetworkError` it will be retried 30 seconds later with a subsequent
+retry 30 seconds after that. If it throws a `SystemCallError` it will first
+retry 120 seconds later then a subsequent retry attempt 240 seconds later.  If
+the job fails due to a `NetworkError`, Resque would retry the job in 30 seconds.
+If the job fails a second time, this time due to a `SystemCallError`, the next
+retry would occur 240 seconds later as specified in the `SystemCallError`
+array defined in `@retry_exceptions`.
 
 ### <a name="fail_fast"></a> Fail Fast For Specific Exceptions
 
@@ -511,7 +522,7 @@ class ExampleJob
 
   def self.work(*args)
     user_id, user_mode, record_id = *args
-    
+
     Resque.enqueue_to(
       target_queue_for_args(user_id, user_mode, record_id),
       self,
@@ -531,7 +542,7 @@ class ExampleJob
     user_id, user_mode, record_id = *args
 
     if user_mode
-      'high
+      'high'
     else
       'low'
     end
@@ -585,7 +596,7 @@ end
 
 This saves you from having to run a "house cleaning" or "errand" job.
 
-The expiary timeout is "pushed forward" or "touched" after each failure to
+The expiry timeout is "pushed forward" or "touched" after each failure to
 ensure it's not expired too soon.
 
 ### <a name="callbacks"></a> Try Again and Give Up Callbacks

data/lib/resque-retry/server.rb

@@ -66,15 +66,23 @@ module ResqueRetry
       # cancels job retry
       def cancel_retry(job)
         klass = get_class(job)
-        retry_key = retry_key_for_job(job)
-        Resque.remove_delayed(klass, *job['args'])
-        Resque.redis.del("failure-#{retry_key}")
-        Resque.redis.del(retry_key)
+        if klass
+          retry_key = retry_key_for_job(job)
+          Resque.remove_delayed(klass, *job['args'])
+          Resque.redis.del("failure-#{retry_key}")
+          Resque.redis.del(retry_key)
+        else
+          raise 'cannot cancel, job not found'
+        end
       end
 
       private
       def get_class(job)
-        Resque::Job.new(nil, nil).constantize(job['class'])
+        begin
+          Resque::Job.new(nil, nil).constantize(job['class'])
+        rescue
+          nil
+        end
       end
     end
 

data/lib/resque-retry/version.rb

@@ -1,3 +1,3 @@
 module ResqueRetry
-  VERSION = '1.7.2'
+  VERSION = '1.7.6'
 end

data/lib/resque/failure/multiple_with_retry_suppression.rb

@@ -42,10 +42,25 @@ module Resque
           )
 
           cleanup_retry_failure_log!
-          super
-        elsif retry_delay > 0
+          return super
+        end
+
+        # some plugins define retry_delay and have it take no arguments, so rather than break those,
+        # we'll just check here to see whether it takes the additional exception class argument or not
+        # we also allow all job args to be passed to a custom `retry_delay` method
+        retry_delay_arity = klass.method(:retry_delay).arity
+
+        calculated_retry_delay = if [-2, 2].include?(retry_delay_arity)
+          klass.retry_delay(exception.class, *args)
+        elsif [-1, 1].include?(retry_delay_arity)
+          klass.retry_delay(exception.class)
+        else
+          klass.retry_delay
+        end
+
+        if calculated_retry_delay > 0
           log_message(
-            "retry_delay: #{retry_delay} > 0 - saving details in Redis",
+            "retry_delay: #{calculated_retry_delay} > 0 - saving details in Redis",
             args,
             exception
           )
@@ -63,12 +78,12 @@ module Resque
 
           Resque.redis.setex(
             failure_key,
-            2 * retry_delay,
+            2 * calculated_retry_delay,
             data
           )
         else
           log_message(
-            "retry_delay: #{retry_delay} <= 0 - ignoring",
+            "retry_delay: #{calculated_retry_delay} <= 0 - ignoring",
             args,
             exception
           )
@@ -117,10 +132,6 @@ module Resque
         Resque::Job.new(nil, nil).constantize(payload['class'])
       end
 
-      def retry_delay
-        klass.retry_delay
-      end
-
       def retry_key
         klass.redis_retry_key(*payload['args'])
       end
@@ -132,7 +143,17 @@ module Resque
       end
 
       def retrying?
-        Resque.redis.exists(retry_key)
+        redis_key_exists?(retry_key)
+      end
+
+      private
+
+      def redis_key_exists?(key)
+        if Resque.redis.respond_to?(:exists?)
+          Resque.redis.exists?(key)
+        else
+          ![false, 0].include?(Resque.redis.exists(key) || false)
+        end
       end
     end
   end

data/lib/resque/plugins/exponential_backoff.rb

@@ -80,10 +80,11 @@ module Resque
 
       # Selects the delay from the backoff strategy
       #
+      # @param _ [Exception] unused exception argument for signature parity
       # @return [Number] seconds to delay until the next retry.
       #
       # @api private
-      def retry_delay
+      def retry_delay(_ = nil)
         delay = backoff_strategy[retry_attempt] || backoff_strategy.last
         # if the values are the same don't bother generating a random number, if
         # the delta is zero, some platforms will raise an error

data/lib/resque/plugins/retry.rb

@@ -107,11 +107,27 @@ module Resque
       # A retry limit of 0 will *never* retry.
       # A retry limit of -1 or below will retry forever.
       #
+      # The default value is: `1` or in the case of where `@retry_exceptions` is
+      # specified, and it contains one or more `Array` values, the maximum
+      # length will be used (e.g. `@retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }`
+      # would return `2` because `SystemCallError` _should_ be attempted at
+      # least twice to respect the specified configuration).
+      #
       # @return [Fixnum]
       #
       # @api public
       def retry_limit
-        @retry_limit ||= 1
+        @retry_limit ||= begin
+          default_retry_limit = 1
+          if instance_variable_defined?(:@retry_exceptions) && @retry_exceptions.is_a?(Hash)
+            @retry_exceptions.values.each do |value|
+              if value.is_a?(Array) && value.length > default_retry_limit
+                default_retry_limit = value.length
+              end
+            end
+          end
+          default_retry_limit
+        end
       end
 
       # Number of retry attempts used to try and perform the job

data/test/exponential_backoff_test.rb

@@ -128,4 +128,11 @@ class ExponentialBackoffTest < Minitest::Test
     assert_equal 4, Resque.info[:failed],     'failed jobs'
     assert_equal 0, Resque.info[:pending],    'pending jobs'
   end
+
+  def test_backoff_with_expiration
+    Resque.redis.expects(:expire)
+
+    Resque.enqueue(ExponentialBackoffWithExpiryJob)
+    perform_next_job(@worker)
+  end
 end

data/test/multiple_failure_test.rb

@@ -1,4 +1,4 @@
-require 'test_helper'
+require_relative './test_helper'
 
 # Mock failure backend for testing MultipleWithRetrySuppression
 class MockFailureBackend < Resque::Failure::Base
@@ -25,8 +25,8 @@ class MultipleFailureTest < Minitest::Test
     Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
   end
 
-  def failure_key_for(klass)
-    'failure-' + klass.redis_retry_key([])
+  def failure_key_for(klass, *args)
+    Resque::Failure::MultipleWithRetrySuppression.failure_key(klass.redis_retry_key(args))
   end
 
   def test_failure_is_passed_on_when_job_class_not_found
@@ -53,6 +53,22 @@ class MultipleFailureTest < Minitest::Test
     assert Resque.redis.exists(key)
   end
 
+  def test_retry_delay_is_calculated_with_custom_calculation
+    delay = 5
+    Resque.enqueue(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
+    perform_next_job(@worker)
+
+    key = failure_key_for(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
+    ttl = Resque.redis.ttl(key)
+    assert Resque.redis.exists(key)
+    assert MockFailureBackend.errors.size == 0
+
+    # expiration on failure_key is set to 2x the delay
+    # to ensure the customized delay is properly calculated using
+    # dynamic retry_delay method on the job
+    assert ttl > delay && delay <= (delay * 2)
+  end
+
   def test_retry_key_splatting_args
     # were expecting this to be called three times:
     # - once when we queue the job to try again
@@ -141,6 +157,20 @@ class MultipleFailureTest < Minitest::Test
     end
   end
 
+  def test_redis_exists_returns_integer
+    Resque.enqueue(RetryDefaultsJob)
+    original = Redis.exists_returns_integer
+    Redis.exists_returns_integer = true
+
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    Redis.exists_returns_integer = original
+
+    assert_equal 1, MockFailureBackend.errors.size
+  end
+
   def teardown
     Resque::Failure.backend = @old_failure_backend
   end

data/test/retry_exception_delay_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class RetryExeptionDelayTest < Minitest::Test
+class RetryExceptionDelayTest < Minitest::Test
   def setup
     Resque.redis.flushall
     @worker = Resque::Worker.new(:testing)
@@ -43,4 +43,12 @@ class RetryExeptionDelayTest < Minitest::Test
     assert_in_delta (start_time + 10), delayed[1], 1.00, '2nd retry delay timestamp'
     assert_in_delta (start_time + 15), delayed[2], 1.00, '3rd retry delay timestamp'
   end
+
+  def test_retry_delay_per_exception_multiple_delay_no_retry_limit_specified
+    # For this job-type there are 3 `Exception` types defined: `Exception`,
+    # `StandardError` and `Timeout::Error` -- their array-lengths are are 1, 3
+    # and 5 (respectively). We expect the `retry_limit` to default to the
+    # maximum length of the `Array` of delays or 1 (in this case: 5)
+    assert_equal 5, PerExceptionClassRetryCountArrayNoRetryLimitSpecifiedJob.retry_limit
+  end
 end

data/test/test_helper.rb

@@ -1,6 +1,8 @@
 dir = File.dirname(File.expand_path(__FILE__))
 $LOAD_PATH.unshift dir + '/../lib'
+
 $TESTING = true
+$VERBOSE = ENV['VERBOSE'] != 'true' ? nil : true
 
 # Run code coverage in MRI 1.9 only.
 if RUBY_VERSION >= '1.9' && RUBY_ENGINE == 'ruby'
@@ -20,25 +22,29 @@ require 'mocha/setup'
 require 'resque-retry'
 require dir + '/test_jobs'
 
-# make sure we can run redis-server
-if !system('which redis-server')
-  puts '', "** `redis-server` was not found in your PATH"
-  abort ''
-end
+if ENV['CI'] != 'true'
+  # make sure we can run redis-server
+  if !system('which redis-server')
+    puts '', "** `redis-server` was not found in your PATH"
+    abort ''
+  end
 
-# make sure we can shutdown the server using cli.
-if !system('which redis-cli')
-  puts '', "** `redis-cli` was not found in your PATH"
-  abort ''
-end
+  # make sure we can shutdown the server using cli.
+  if !system('which redis-cli')
+    puts '', "** `redis-cli` was not found in your PATH"
+    abort ''
+  end
 
-# This code is run after all the tests have finished running to ensure that the
-# Redis server is shutdowa
-Minitest.after_run { `redis-cli -p 9736 shutdown nosave` }
+  # This code is run after all the tests have finished running to ensure that the
+  # Redis server is shutdowa
+  Minitest.after_run { `redis-cli -p 9736 shutdown nosave` }
 
-puts "Starting redis for testing at localhost:9736..."
-`redis-server #{dir}/redis-test.conf`
-Resque.redis = '127.0.0.1:9736'
+  puts "Starting redis for testing at localhost:9736..."
+  `redis-server #{dir}/redis-test.conf`
+  Resque.redis = '127.0.0.1:9736'
+else
+  Resque.redis = '127.0.0.1:6379'
+end
 
 # Test helpers
 class Minitest::Test
@@ -65,11 +71,11 @@ class Minitest::Test
 
   def delayed_jobs
     # The double-checks here are so that we won't blow up if the config stops using redis-namespace
-    timestamps = Resque.redis.zrange("resque:delayed_queue_schedule", 0, -1) + 
+    timestamps = Resque.redis.zrange("resque:delayed_queue_schedule", 0, -1) +
                  Resque.redis.zrange("delayed_queue_schedule", 0, -1)
 
     delayed_jobs_as_json = timestamps.map do |timestamp|
-      Resque.redis.lrange("resque:delayed:#{timestamp}", 0, -1) + 
+      Resque.redis.lrange("resque:delayed:#{timestamp}", 0, -1) +
       Resque.redis.lrange("delayed:#{timestamp}", 0, -1)
     end.flatten
 

data/test/test_jobs.rb

@@ -254,6 +254,12 @@ class ExponentialBackoffWithRetryDelayMultiplicandMinAndMaxJob < RetryDefaultsJo
   @retry_delay_multiplicand_max = 3.0
 end
 
+class ExponentialBackoffWithExpiryJob < RetryDefaultsJob
+  extend Resque::Plugins::ExponentialBackoff
+  @queue = :testing
+  @expire_retry_key_after = 60 * 60
+end
+
 class InvalidRetryDelayMaxConfigurationJob
   @queue = :testing
   @retry_delay_multiplicand_max = 0.9
@@ -576,6 +582,17 @@ class PerExceptionClassRetryCountArrayJob
   end
 end
 
+class PerExceptionClassRetryCountArrayNoRetryLimitSpecifiedJob
+  extend Resque::Plugins::Retry
+
+  @queue = :testing
+  @retry_exceptions = { Exception => 11, RuntimeError => [5, 10, 15],  Timeout::Error => [2, 4, 6, 8, 10] }
+
+  def self.perform
+    raise RuntimeError, 'I always fail with a RuntimeError'
+  end
+end
+
 # We can't design a job to fail during connect, see perform_next_job_fail_on_reconnect
 class FailsDuringConnectJob < RetryDefaultsJob
   @queue = :testing

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: resque-retry
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.7.6
 platform: ruby
 authors:
 - Luke Antins
 - Ryan Carver
 - Jonathan W. Zaleski
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-21 00:00:00.000000000 Z
+date: 2021-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -161,8 +161,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
 - HISTORY.md
 - LICENSE
@@ -208,7 +208,7 @@ homepage: http://github.com/lantins/resque-retry
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -223,8 +223,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: A resque plugin; provides retry, delay and exponential backoff support for
   resque jobs.

data/.travis.yml

@@ -1,14 +0,0 @@
-language: ruby
-
-sudo: false
-
-matrix:
-  allow_failures:
-    - rvm: jruby-9.1.9.0
-
-rvm:
-  - 2.3.8
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-  - jruby-9.1.9.0