resque-retry 1.1.4 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1b6a97e99754f141a99af11bd0d77f5f73d8010
-  data.tar.gz: 2c6e42f2294034e932b4a12998770214a0eed1ea
+  metadata.gz: 3424ef6139214b3f9ec367393e92e2da6ac888eb
+  data.tar.gz: 85ba61ab491cadba2783e34e3af7d05953e02f53
 SHA512:
-  metadata.gz: d748027d00df3b099e92cac7123baedccca96f0240a5f17c27f15277a57c0c527a3207e277950ce42b57a3f440791cf75e2245d4a920db35f897d126f932a895
-  data.tar.gz: 56ce881aff8343f9d381861aa38a061a6bf80eaf3e099f90d6a41ca6a3ca3fef52cb30cb12a3014809400d6c52b5f1f1c0103fd8ee6454878b0f512e0f8c2daf
+  metadata.gz: 6424ef718b680bfd61cc7fa47ddc1c09ae397c5a9838fbeee01fa4af5ff79001c06052f982ab517aa8f2db73d7e1f99323ca41ad4df54d328897f104ea07bfaf
+  data.tar.gz: 840b6f3402ee1bf8e708ab78d6f7f376aac96f0ef843ff5d08c9668ed1771d69b6a51a819216be8f5611013e0cfdda9f30575ec1d330ec97af304f359298963e

data/.gitignore

@@ -12,3 +12,5 @@ stdout
 tags
 tags.*
 *.gem
+.ruby-version
+.ruby-gemset

data/.travis.yml

@@ -1,9 +1,16 @@
-# configuration settings for http://travis-ci.org
-
 language: ruby
+
+matrix:
+  allow_failures:
+    - rvm: jruby-19mode
+    - rvm: jruby-20mode
+    - rvm: rbx
+
 rvm:
-  - 2.1.0
-  - 2.0.0
   - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.1.1
   - jruby-19mode
+  - jruby-20mode
   - rbx

data/HISTORY.md

@@ -1,5 +1,11 @@
 ## HEAD
 
+## 1.2.0 (2014-05-19)
+
+* Fixed scenario where job does not get retried correctly when `perform` is not called as expected.
+* Feature: Optional `@expire_retry_key_after` settings; expires retry counters from redis to save you cleaning up stale state.
+* Feature: Expose inner-workings of plugin through debug messages using `Resque.logger` (when logging level is Logger:DEBUG).
+
 ## 1.1.4 (2014-03-17)
 
 * Fixed displaying retry information in resque web interface, caused by `Resque::Helpers` being deprecated.

data/README.md

@@ -26,35 +26,39 @@ If you're using [Bundler][bundler] to manage your dependencies, you should add `
 'resque-retry'` to your projects `Gemfile`.
 
 Add this to your `Rakefile`:
-
-    require 'resque/tasks'
-    require 'resque_scheduler/tasks'
+```ruby
+require 'resque/tasks'
+require 'resque_scheduler/tasks'
+```
 
 The delay between retry attempts is provided by [resque-scheduler][rqs].
 You'll want to run the scheduler process, otherwise delayed retry attempts
 will never perform:
-
-    $ rake resque:scheduler
+```
+$ rake resque:scheduler
+```
 
 Use the plugin:
+```ruby
+require 'resque-retry'
 
-    require 'resque-retry'
+class ExampleRetryJob
+  extend Resque::Plugins::Retry
+  @queue = :example_queue
 
-    class ExampleRetryJob
-      extend Resque::Plugins::Retry
-      @queue = :example_queue
+  @retry_limit = 3
+  @retry_delay = 60
 
-      @retry_limit = 3
-      @retry_delay = 60
-
-      def self.perform(*args)
-        # your magic/heavy lifting goes here.
-      end
-    end
+  def self.perform(*args)
+    # your magic/heavy lifting goes here.
+  end
+end
+```
 
 Then start up a resque worker as normal:
-
-    $ QUEUE=* rake resque:work
+```
+$ QUEUE=* rake resque:work
+```
 
 Now if you ExampleRetryJob fails, it will be retried 3 times, with a 60 second
 delay between attempts.
@@ -79,14 +83,15 @@ actually completed successfully just by just using the resque-web interface.
 `MultipleWithRetrySuppression` is a multiple failure backend, with retry suppression.
 
 Here's an example, using the Redis failure backend:
+```ruby
+require 'resque-retry'
+require 'resque/failure/redis'
 
-    require 'resque-retry'
-    require 'resque/failure/redis'
+# require your jobs & application code.
 
-    # require your jobs & application code.
-
-    Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
-    Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
+Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
+Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
+```
 
 If a job fails, but **can and will** retry, the failure details wont be
 logged in the Redis failed queue *(visible via resque-web)*.
@@ -103,13 +108,14 @@ The new Retry tab displays delayed jobs with retry information; the number of
 attempts and the exception details from the last failure.
 
 Make sure you include this in your `config.ru` or similar file:
+```ruby
+require 'resque-retry'
+require 'resque-retry/server'
 
-    require 'resque-retry'
-    require 'resque-retry/server'
-
-    # require your jobs & application code.
+# require your jobs & application code.
 
-    run Resque::Server.new
+run Resque::Server.new
+```
 
 Retry Options & Logic
 ---------------------
@@ -123,35 +129,37 @@ some ideas =), adapt for your own usage and feel free to pick and mix!
 ### Retry Defaults
 
 Retry the job **once** on failure, with zero delay.
+```ruby
+require 'resque-retry'
 
-    require 'resque-retry'
+class DeliverWebHook
+  extend Resque::Plugins::Retry
+  @queue = :web_hooks
 
-    class DeliverWebHook
-      extend Resque::Plugins::Retry
-      @queue = :web_hooks
-
-      def self.perform(url, hook_id, hmac_key)
-        heavy_lifting
-      end
-    end
+  def self.perform(url, hook_id, hmac_key)
+    heavy_lifting
+  end
+end
+```
 
 When a job runs, the number of retry attempts is checked and incremented
 in Redis. If your job fails, the number of retry attempts is used to
 determine if we can requeue the job for another go.
 
 ### Custom Retry
+```ruby
+class DeliverWebHook
+  extend Resque::Plugins::Retry
+  @queue = :web_hooks
 
-    class DeliverWebHook
-      extend Resque::Plugins::Retry
-      @queue = :web_hooks
-
-      @retry_limit = 10
-      @retry_delay = 120
+  @retry_limit = 10
+  @retry_delay = 120
 
-      def self.perform(url, hook_id, hmac_key)
-        heavy_lifting
-      end
-    end
+  def self.perform(url, hook_id, hmac_key)
+    heavy_lifting
+  end
+end
+```
 
 The above modification will allow your job to retry up to 10 times, with
 a delay of 120 seconds, or 2 minutes between retry attempts.
@@ -164,17 +172,18 @@ more special.
 Sometimes it is useful to delay the worker that failed a job attempt, but
 still requeue the job for immediate processing by other workers. This can be
 done with `@sleep_after_requeue`:
+```ruby
+class DeliverWebHook
+  extend Resque::Plugins::Retry
+  @queue = :web_hooks
 
-    class DeliverWebHook
-      extend Resque::Plugins::Retry
-      @queue = :web_hooks
-
-      @sleep_after_requeue = 5
+  @sleep_after_requeue = 5
 
-      def self.perform(url, hook_id, hmac_key)
-        heavy_lifting
-      end
-    end
+  def self.perform(url, hook_id, hmac_key)
+    heavy_lifting
+  end
+end
+```
 
 This retries the job once and causes the worker that failed to sleep for 5
 seconds after requeuing the job. If there are multiple workers in the system
@@ -192,24 +201,26 @@ dynamically.
 ### Exponential Backoff
 
 Use this if you wish to vary the delay between retry attempts:
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::ExponentialBackoff
+  @queue = :mt_messages
 
-    class DeliverSMS
-      extend Resque::Plugins::ExponentialBackoff
-      @queue = :mt_messages
-
-      def self.perform(mt_id, mobile_number, message)
-        heavy_lifting
-      end
-    end
+  def self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+```
 
 **Default Settings**
+```
+key: m = minutes, h = hours
 
-    key: m = minutes, h = hours
-
-                  no delay, 1m, 10m,   1h,    3h,    6h
-    @backoff_strategy = [0, 60, 600, 3600, 10800, 21600]
-    @retry_delay_multiplicand_min = 1.0
-    @retry_delay_multiplicand_max = 1.0
+              no delay, 1m, 10m,   1h,    3h,    6h
+@backoff_strategy = [0, 60, 600, 3600, 10800, 21600]
+@retry_delay_multiplicand_min = 1.0
+@retry_delay_multiplicand_max = 1.0
+```
 
 The first delay will be 0 seconds, the 2nd will be 60 seconds, etc...
 Again, tweak to your own needs.
@@ -228,17 +239,18 @@ them all retried on the same schedule.
 
 The default will allow a retry for any type of exception. You may change
 it so only specific exceptions are retried using `retry_exceptions`:
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::Retry
+  @queue = :mt_messages
 
-    class DeliverSMS
-      extend Resque::Plugins::Retry
-      @queue = :mt_messages
-
-      @retry_exceptions = [NetworkError]
+  @retry_exceptions = [NetworkError]
 
-      def self.perform(mt_id, mobile_number, message)
-        heavy_lifting
-      end
-    end
+  def self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+```
 
 The above modification will **only** retry if a `NetworkError` (or subclass)
 exception is thrown.
@@ -248,17 +260,18 @@ types. You may optionally set `@retry_exceptions` to a hash where the keys are
 your specific exception classes to retry on, and the values are your retry
 delays in seconds or an array of retry delays to be used similar to
 exponential backoff.
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::Retry
+  @queue = :mt_messages
 
-    class DeliverSMS
-      extend Resque::Plugins::Retry
-      @queue = :mt_messages
+  @retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }
 
-      @retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }
-
-      def self.perform(mt_id, mobile_number, message)
-        heavy_lifting
-      end
-    end
+  def self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+```
 
 In the above example, Resque would retry any `DeliverSMS` jobs which throw a
 `NetworkError` or `SystemCallError`. If the job throws a `NetworkError` it
@@ -269,17 +282,18 @@ retry 120 seconds later then subsequent retry attempts 240 seconds later.
 
 The default will allow a retry for any type of exception. You may change
 it so specific exceptions fail immediately by using `fatal_exceptions`:
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::Retry
+  @queue = :mt_divisions
 
-    class DeliverSMS
-      extend Resque::Plugins::Retry
-      @queue = :mt_divisions
+  @fatal_exceptions = [NetworkError]
 
-      @fatal_exceptions = [NetworkError]
-
-      def self.perform(mt_id, mobile_number, message)
-        heavy_lifting
-      end
-    end
+  def self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+```
 
 In the above example, Resque would retry any `DeliverSMS` jobs that throw any
 type of error other than `NetworkError`. If the job throws a `NetworkError` it
@@ -288,25 +302,26 @@ will be marked as "failed" immediately.
 ### Custom Retry Criteria Check Callbacks
 
 You may define custom retry criteria callbacks:
-
-    class TurkWorker
-      extend Resque::Plugins::Retry
-      @queue = :turk_job_processor
-
-      @retry_exceptions = [NetworkError]
-
-      retry_criteria_check do |exception, *args|
-        if exception.message =~ /InvalidJobId/
-          false # don't retry if we got passed a invalid job id.
-        else
-          true  # its okay for a retry attempt to continue.
-        end
-      end
-
-      def self.perform(job_id)
-        heavy_lifting
-      end
+```ruby
+class TurkWorker
+  extend Resque::Plugins::Retry
+  @queue = :turk_job_processor
+
+  @retry_exceptions = [NetworkError]
+
+  retry_criteria_check do |exception, *args|
+    if exception.message =~ /InvalidJobId/
+      false # don't retry if we got passed a invalid job id.
+    else
+      true  # its okay for a retry attempt to continue.
     end
+  end
+
+  def self.perform(job_id)
+    heavy_lifting
+  end
+end
+```
 
 Similar to the previous example, this job will retry if either a
 `NetworkError` (or subclass) exception is thrown **or** any of the callbacks
@@ -319,20 +334,21 @@ job should retry.
 
 You may override `args_for_retry`, which is passed the current
 job arguments, to modify the arguments for the next retry attempt.
-
-    class DeliverViaSMSC
-      extend Resque::Plugins::Retry
-      @queue = :mt_smsc_messages
-
-      # retry using the emergency SMSC.
-      def self.args_for_retry(smsc_id, mt_message)
-        [999, mt_message]
-      end
-
-      self.perform(smsc_id, mt_message)
-        heavy_lifting
-      end
-    end
+```ruby
+class DeliverViaSMSC
+  extend Resque::Plugins::Retry
+  @queue = :mt_smsc_messages
+
+  # retry using the emergency SMSC.
+  def self.args_for_retry(smsc_id, mt_message)
+    [999, mt_message]
+  end
+
+  self.perform(smsc_id, mt_message)
+    heavy_lifting
+  end
+end
+```
 
 ### Job Retry Identifier/Key
 
@@ -347,19 +363,48 @@ By default the key uses this format:
 `resque-retry:<job class name>:<retry_identifier>`.
 
 Or you can define the entire key by overriding `redis_retry_key`.
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::Retry
+  @queue = :mt_messages
 
-    class DeliverSMS
-      extend Resque::Plugins::Retry
-      @queue = :mt_messages
+  def self.retry_identifier(mt_id, mobile_number, message)
+    "#{mobile_number}:#{mt_id}"
+  end
 
-      def self.retry_identifier(mt_id, mobile_number, message)
-        "#{mobile_number}:#{mt_id}"
-      end
+  self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+```
 
-      self.perform(mt_id, mobile_number, message)
-        heavy_lifting
-      end
-    end
+### Expire Retry Counters From Redis
+
+Allow the Redis to expire stale retry counters from the database by setting
+`@expire_retry_key_after`:
+
+```ruby
+class DeliverSMS
+  extend Resque::Plugins::Retry
+  @queue = :mt_messages
+  @expire_retry_key_after = 3600 # expire key after `retry_delay` plus 1 hour
+
+  self.perform(mt_id, mobile_number, message)
+    heavy_lifting
+  end
+end
+
+```
+
+This saves you from having to run a "house cleaning" or "errand" job.
+
+The expiary timeout is "pushed forward" or "touched" after each failure to
+ensure its not expired too soon.
+
+### Debug Plugging Logging
+
+The inner-workings of the plugin are output to the Resque [Logger](https://github.com/resque/resque/wiki/Logging)
+when `Resque.logger.level` is set to `Logger::DEBUG`.
 
 Contributing/Pull Requests
 --------------------------

data/lib/resque-retry/version.rb

@@ -1,3 +1,3 @@
 module ResqueRetry
-  VERSION = '1.1.4'
+  VERSION = '1.2.0'
 end

data/lib/resque/failure/multiple_with_retry_suppression.rb

@@ -1,4 +1,5 @@
 require 'resque/failure/multiple'
+require 'resque/plugins/retry/logging'
 
 module Resque
   module Failure
@@ -17,6 +18,8 @@ module Resque
     #   Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
     #
     class MultipleWithRetrySuppression < Multiple
+      include Resque::Plugins::Retry::Logging
+
       # Called when the job fails
       #
       # If the job will retry, suppress the failure from the other backends.
@@ -25,10 +28,17 @@ module Resque
       #
       # @api private
       def save
-        if !(retryable? && retrying?)
+        log_message 'failure backend save', args_from(payload), exception
+
+        retryable = retryable?
+        job_being_retried = retryable && retrying?
+
+        if !job_being_retried
+          log_message "#{retryable ? '' : 'non-'}retriable job is not being retried - sending failure to superclass", args_from(payload), exception
           cleanup_retry_failure_log!
           super
         elsif retry_delay > 0
+          log_message "retry_delay: #{retry_delay} > 0 - saving details in Redis", args_from(payload), exception
           data = {
             :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
             :payload   => payload,
@@ -40,6 +50,8 @@ module Resque
           }
 
           Resque.redis.setex(failure_key, 2*retry_delay, Resque.encode(data))
+        else
+          log_message "retry_delay: #{retry_delay} <= 0 - ignoring", args_from(payload), exception
         end
       end
 
@@ -82,6 +94,10 @@ module Resque
       def cleanup_retry_failure_log!
         Resque.redis.del(failure_key) if retryable?
       end
+
+      def args_from(payload)
+        (payload || {})['args']
+      end
     end
   end
 end

data/lib/resque/plugins/retry.rb

@@ -1,4 +1,5 @@
 require 'digest/sha1'
+require 'resque/plugins/retry/logging'
 
 module Resque
   module Plugins
@@ -35,6 +36,7 @@ module Resque
     #   end
     #
     module Retry
+      include Resque::Plugins::Retry::Logging
 
       # Raised if the retry-strategy cannot be determined or has conflicts
       #
@@ -45,7 +47,7 @@ module Resque
       #
       # @api private
       def self.extended(receiver)
-        if receiver.instance_variable_get("@fatal_exceptions") && receiver.instance_variable_get("@retry_exceptions")
+        if receiver.instance_variable_get('@fatal_exceptions') && receiver.instance_variable_get('@retry_exceptions')
           raise AmbiguousRetryStrategyException.new(%{You can't define both "@fatal_exceptions" and "@retry_exceptions"})
         end
       end
@@ -55,7 +57,7 @@ module Resque
       # @api private
       def inherited(subclass)
         super(subclass)
-        subclass.instance_variable_set("@retry_criteria_checks", retry_criteria_checks.dup)
+        subclass.instance_variable_set('@retry_criteria_checks', retry_criteria_checks.dup)
       end
 
       # @abstract You may override to implement a custom retry identifier,
@@ -81,7 +83,7 @@ module Resque
       #
       # @api public
       def redis_retry_key(*args)
-        ['resque-retry', name, retry_identifier(*args)].compact.join(":").gsub(/\s/, '')
+        ['resque-retry', name, retry_identifier(*args)].compact.join(':').gsub(/\s/, '')
       end
 
       # Maximum number of retrys we can attempt to successfully perform the job
@@ -230,17 +232,25 @@ module Resque
       # @api public
       def retry_criteria_valid?(exception, *args)
         # if the retry limit was reached, dont bother checking anything else.
-        return false if retry_limit_reached?
+        if retry_limit_reached?
+          log_message 'retry limit reached', args, exception
+          return false 
+        end
 
         # We always want to retry if the exception matches.
-        should_retry = retry_exception?(exception)
-
-        # call user retry criteria check blocks.
-        retry_criteria_checks.each do |criteria_check|
-          should_retry ||= !!instance_exec(exception, *args, &criteria_check)
+        retry_based_on_exception = retry_exception?(exception)
+        log_message "Exception is #{retry_based_on_exception ? '' : 'not '}sufficient for a retry", args, exception
+
+        retry_based_on_criteria = false
+        unless retry_based_on_exception
+          # call user retry criteria check blocks.
+          retry_criteria_checks.each do |criteria_check|
+            retry_based_on_criteria ||= !!instance_exec(exception, *args, &criteria_check)
+          end
         end
+        log_message "user retry criteria is #{retry_based_on_criteria ? '' : 'not '}sufficient for a retry", args, exception
 
-        should_retry
+        retry_based_on_exception || retry_based_on_criteria
       end
 
       # Retry criteria checks
@@ -297,11 +307,21 @@ module Resque
       #
       # @api private
       def try_again(exception, *args)
+        log_message 'try_again', args, exception
         # some plugins define retry_delay and have it take no arguments, so rather than break those,
         # we'll just check here to see whether it takes the additional exception class argument or not
         temp_retry_delay = ([-1, 1].include?(method(:retry_delay).arity) ? retry_delay(exception.class) : retry_delay)
 
         retry_in_queue = retry_job_delegate ? retry_job_delegate : self
+        log_message "retry delay: #{temp_retry_delay} for class: #{retry_in_queue}", args, exception
+
+        # remember that this job is now being retried. before_perform_retry will increment
+        # this so it represents the retry count, and MultipleWithRetrySuppression uses
+        # the existence of this to determine if the job should be sent to the 
+        # parent failure backend (e.g. failed queue) or not.  Removing this means
+        # jobs that fail before ::perform will be both retried and sent to the failed queue.
+        Resque.redis.setnx(redis_retry_key(*args), -1)
+
         if temp_retry_delay <= 0
           # If the delay is 0, no point passing it through the scheduler
           Resque.enqueue(retry_in_queue, *args_for_retry(*args))
@@ -322,12 +342,15 @@ module Resque
       #
       # @api private
       def before_perform_retry(*args)
+        log_message 'before_perform_retry', args
         @on_failure_retry_hook_already_called = false
 
         # store number of retry attempts.
         retry_key = redis_retry_key(*args)
         Resque.redis.setnx(retry_key, -1)             # default to -1 if not set.
         @retry_attempt = Resque.redis.incr(retry_key) # increment by 1.
+        log_message "attempt: #{@retry_attempt} set in Redis", args
+        Resque.redis.expire(retry_key, @retry_delay.to_i + @expire_retry_key_after.to_i) if @expire_retry_key_after
       end
 
       # Resque after_perform hook
@@ -336,6 +359,7 @@ module Resque
       #
       # @api private
       def after_perform_retry(*args)
+        log_message 'after_perform_retry, clearing retry key', args
         clean_retry_key(*args)
       end
 
@@ -350,11 +374,16 @@ module Resque
       #
       # @api private
       def on_failure_retry(exception, *args)
-        return if @on_failure_retry_hook_already_called
+        log_message 'on_failure_retry', args, exception
+        if @on_failure_retry_hook_already_called
+          log_message 'on_failure_retry_hook_already_called', args, exception
+          return 
+        end
 
         if retry_criteria_valid?(exception, *args)
           try_again(exception, *args)
         else
+          log_message 'retry criteria not sufficient for retry', args, exception
           clean_retry_key(*args)
         end
 
@@ -381,9 +410,9 @@ module Resque
       #
       # @api private
       def clean_retry_key(*args)
+        log_message 'clean_retry_key', args
         Resque.redis.del(redis_retry_key(*args))
       end
-
     end
   end
 end

data/lib/resque/plugins/retry/logging.rb

@@ -0,0 +1,23 @@
+module Resque
+  module Plugins
+    module Retry
+      module Logging
+        # Log messages through the Resque logger (DEBUG level).
+        # Generally not for application logging-just for inner-workings of
+        # Resque and plugins.
+        #
+        # @param [String] message to log
+        # @param [Object] args of the resque job in context
+        # @param [Object] exception that might be causing a retry
+        #
+        # @api private
+        def log_message(message, args=nil, exception=nil)
+          return unless Resque.logger
+
+          exception_portion = exception.nil? ? '' : " [#{exception.class}/#{exception}]"
+          Resque.logger.debug "resque-retry -- #{args.inspect}#{exception_portion}: #{message}"
+        end
+      end
+    end
+  end
+end

data/resque-retry.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.name = 'resque-retry'
   s.version = ResqueRetry::VERSION
   s.date = Time.now.strftime('%Y-%m-%d')
-  s.authors = ['Luke Antins', 'Ryan Carver']
+  s.authors = ['Luke Antins', 'Ryan Carver', 'Jonathan W. Zaleski']
   s.email = ['luke@lividpenguin.com']
   s.summary = 'A resque plugin; provides retry, delay and exponential backoff support for resque jobs.'
   s.description = <<-EOL

data/test/multiple_failure_test.rb

@@ -54,10 +54,11 @@ class MultipleFailureTest < MiniTest::Unit::TestCase
   end
 
   def test_retry_key_splatting_args
-    # were expecting this to be called twice:
+    # were expecting this to be called three times:
+    # - once when we queue the job to try again
     # - once before the job is executed.
     # - once by the failure backend.
-    RetryDefaultsJob.expects(:redis_retry_key).with({'a' => 1, 'b' => 2}).times(2)
+    RetryDefaultsJob.expects(:redis_retry_key).with({'a' => 1, 'b' => 2}).times(3)
 
     Resque.enqueue(RetryDefaultsJob, {'a' => 1, 'b' => 2})
     perform_next_job(@worker)
@@ -130,6 +131,16 @@ class MultipleFailureTest < MiniTest::Unit::TestCase
     assert_equal 1, MockFailureBackend.errors.size
   end
 
+  def test_failure_with_retry_bumps_key_expire
+    Resque.enqueue(FailFiveTimesWithExpiryJob, 'foo')
+    retry_key = FailFiveTimesWithExpiryJob.redis_retry_key('foo')
+
+    Resque.redis.expects(:expire).times(4).with(retry_key, 3600)
+    4.times do
+      perform_next_job(@worker)
+    end
+  end
+
   def teardown
     Resque::Failure.backend = @old_failure_backend
   end

data/test/retry_test.rb

@@ -1,5 +1,5 @@
 require 'test_helper'
-
+require 'resque/failure/redis'
 require 'digest/sha1'
 
 class RetryTest < MiniTest::Unit::TestCase
@@ -7,6 +7,7 @@ class RetryTest < MiniTest::Unit::TestCase
     Resque.redis.flushall
     @worker = Resque::Worker.new(:testing)
     @worker.register_worker
+    Resque::Failure.backend = Resque::Failure::Redis
   end
 
   def test_resque_plugin_lint
@@ -73,6 +74,20 @@ class RetryTest < MiniTest::Unit::TestCase
     assert_equal 10, Resque.info[:processed], 'processed job'
   end
 
+  def test_failure_before_perform_does_not_both_requeue_and_fail_the_job
+    Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
+    Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
+
+    Resque.enqueue(FailsDuringConnectJob)
+
+    perform_next_job_fail_on_reconnect(@worker)
+
+    assert_equal 1, Resque.info[:processed], 'processed job'
+    assert_equal 0, Resque.info[:failed], "should not be any failed jobs: #{Resque::Failure.all(0, 100)}"
+    assert_equal 0, Resque.info[:pending], 'should not yet be any pending jobs'
+    assert_equal 1, delayed_jobs.size, 'should be a delayed job'
+  end
+
   def test_retry_never_retry
     Resque.enqueue(NeverRetryJob)
     10.times do
@@ -265,4 +280,9 @@ class RetryTest < MiniTest::Unit::TestCase
     assert_equal 13, PerExceptionClassRetryCountJob.retry_delay(Timeout::Error)
   end
 
+  def test_expire_key_set
+    Resque.redis.expects(:expire).once.with(ExpiringJob.redis_retry_key('expiry_test'), 3600)
+    Resque.enqueue(ExpiringJob, 'expiry_test')
+    perform_next_job(@worker)
+  end
 end

data/test/test_helper.rb

@@ -53,6 +53,34 @@ class MiniTest::Unit::TestCase
     worker.done_working
   end
 
+  def perform_next_job_fail_on_reconnect(worker,&block)
+    raise "No work for #{worker}" unless job = worker.reserve
+    worker.working_on job
+
+    # Similar to resque's Worker.work and Worker.process methods
+    begin
+      raise 'error from perform_next_job_fail_on_reconnect'
+      worker.perform(job, &block)
+    rescue Exception => exception
+      worker.report_failed_job(job, exception)
+    ensure
+      worker.done_working
+    end
+  end
+
+  def delayed_jobs
+    # The double-checks here are so that we won't blow up if the config stops using redis-namespace
+    timestamps = Resque.redis.zrange("resque:delayed_queue_schedule", 0, -1) + 
+                 Resque.redis.zrange("delayed_queue_schedule", 0, -1)
+
+    delayed_jobs_as_json = timestamps.map do |timestamp|
+      Resque.redis.lrange("resque:delayed:#{timestamp}", 0, -1) + 
+      Resque.redis.lrange("delayed:#{timestamp}", 0, -1)
+    end.flatten
+
+    delayed_jobs_as_json.map { |json| JSON.parse(json) }
+  end
+
   def clean_perform_job(klass, *args)
     Resque.redis.flushall
     Resque.enqueue(klass, *args)

data/test/test_jobs.rb

@@ -19,6 +19,15 @@ class GoodJob
   end
 end
 
+class ExpiringJob
+  extend Resque::Plugins::Retry
+  @queue = :testing
+  @expire_retry_key_after = 60 * 60
+
+  def self.perform(*args)
+  end
+end
+
 class RetryDefaultSettingsJob
   extend Resque::Plugins::Retry
   @queue = :testing
@@ -135,6 +144,16 @@ class FailFiveTimesJob < RetryDefaultsJob
   end
 end
 
+class FailFiveTimesWithExpiryJob < RetryDefaultsJob
+  @queue = :testing
+  @retry_limit = 6
+  @expire_retry_key_after = 60 * 60
+
+  def self.perform(*args)
+    raise if retry_attempt <= 4
+  end
+end
+
 class ExponentialBackoffJob < RetryDefaultsJob
   extend Resque::Plugins::ExponentialBackoff
   @queue = :testing
@@ -448,3 +467,10 @@ class PerExceptionClassRetryCountArrayJob
     raise RuntimeError, 'I always fail with a RuntimeError'
   end
 end
+
+# We can't design a job to fail during connect, see perform_next_job_fail_on_reconnect
+class FailsDuringConnectJob < RetryDefaultsJob
+  @queue = :testing
+  @retry_limit = 3
+  @retry_delay = 10
+end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: resque-retry
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0
 platform: ruby
 authors:
 - Luke Antins
 - Ryan Carver
+- Jonathan W. Zaleski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-17 00:00:00.000000000 Z
+date: 2014-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -177,6 +178,7 @@ files:
 - lib/resque/failure/multiple_with_retry_suppression.rb
 - lib/resque/plugins/exponential_backoff.rb
 - lib/resque/plugins/retry.rb
+- lib/resque/plugins/retry/logging.rb
 - resque-retry.gemspec
 - test/exponential_backoff_test.rb
 - test/multiple_failure_test.rb