onyx-resque-retry 0.1.0

data/test/multiple_failure_test.rb

@@ -0,0 +1,86 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class MultipleFailureTest < Test::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+
+    @old_failure_backend = Resque::Failure.backend
+    MockFailureBackend.errors = []
+    Resque::Failure::MultipleWithRetrySuppression.classes = [MockFailureBackend]
+    Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
+  end
+
+  def failure_key_for(klass)
+    args = []
+    key = "failure_" + klass.redis_retry_key(args)
+  end
+
+  def test_last_failure_is_saved_in_redis
+    Resque.enqueue(LimitThreeJob)
+    perform_next_job(@worker)
+
+    # I don't like this, but...
+    key = failure_key_for(LimitThreeJob)
+    assert Resque.redis.exists(key)
+  end
+
+  def test_last_failure_removed_from_redis_after_error_limit
+    Resque.enqueue(LimitThreeJob)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    key = failure_key_for(LimitThreeJob)
+    assert Resque.redis.exists(key)
+
+    perform_next_job(@worker)
+    assert !Resque.redis.exists(key)
+  end
+
+  def test_on_success_failure_log_removed_from_redis
+    SwitchToSuccessJob.successful_after = 1
+    Resque.enqueue(SwitchToSuccessJob)
+    perform_next_job(@worker)
+
+    key = failure_key_for(SwitchToSuccessJob)
+    assert Resque.redis.exists(key)
+
+    perform_next_job(@worker)
+    assert !Resque.redis.exists(key), 'key removed on success'
+  ensure
+    SwitchToSuccessJob.reset_defaults
+  end
+
+  def test_errors_are_suppressed_up_to_retry_limit
+    Resque.enqueue(LimitThreeJob)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, MockFailureBackend.errors.size
+  end
+
+  def test_errors_are_logged_after_retry_limit
+    Resque.enqueue(LimitThreeJob)
+    4.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 1, MockFailureBackend.errors.size
+  end
+
+  def test_jobs_without_retry_log_errors
+    5.times do
+      Resque.enqueue(NoRetryJob)
+      perform_next_job(@worker)
+    end
+
+    assert_equal 5, MockFailureBackend.errors.size
+  end
+
+  def teardown
+    Resque::Failure.backend = @old_failure_backend
+  end
+end

data/test/redis-test.conf

@@ -0,0 +1,132 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.
+# shareobjects no
+# shareobjectspoolsize 1024

data/test/resque_test.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# make sure the worlds not fallen from beneith us.
+class ResqueTest < Test::Unit::TestCase
+  def test_resque_version
+    major, minor, patch = Resque::Version.split('.')
+    assert_equal 1, major.to_i, 'major version does not match'
+    assert_operator minor.to_i, :>=, 8, 'minor version is too low'
+  end
+
+  def test_good_job
+    clean_perform_job(GoodJob, 1234, { :cats => :maiow }, [true, false, false])
+
+    assert_equal 0, Resque.info[:failed], 'failed jobs'
+    assert_equal 1, Resque.info[:processed], 'processed job'
+    assert_equal 0, Resque.delayed_queue_schedule_size
+  end
+end

data/test/retry_criteria_test.rb

@@ -0,0 +1,75 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class RetryCriteriaTest < Test::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_retry_criteria_check_should_retry
+    Resque.enqueue(RetryModuleCustomRetryCriteriaCheck)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+  end
+
+  def test_retry_criteria_check_hierarchy_should_not_retry
+    Resque.enqueue(CustomRetryCriteriaCheckDontRetry)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 1, Resque.info[:failed], 'failed jobs'
+    assert_equal 1, Resque.info[:processed], 'processed job'
+  end
+
+  def test_retry_criteria_check_hierarchy_should_retry
+    Resque.enqueue(CustomRetryCriteriaCheckDoRetry)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+  end
+
+  def test_retry_criteria_check_multiple_never_retry
+    Resque.enqueue(CustomRetryCriteriaCheckMultipleFailTwice, 'dont')
+    6.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 1, Resque.info[:failed], 'failed jobs'
+    assert_equal 1, Resque.info[:processed], 'processed job'
+  end
+
+  def test_retry_criteria_check_multiple_do_retry
+    Resque.enqueue(CustomRetryCriteriaCheckMultipleFailTwice, 'do')
+    6.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 3, Resque.info[:processed], 'processed job'
+  end
+
+  def test_retry_criteria_check_multiple_do_retry_again
+    Resque.enqueue(CustomRetryCriteriaCheckMultipleFailTwice, 'do_again')
+    6.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 3, Resque.info[:processed], 'processed job'
+  end
+end

data/test/retry_inheriting_checks_test.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class RetryInheritingChecksTest < Test::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_default_job_has_one_exception
+    assert_equal 0, RetryDefaultsJob.retry_criteria_checks.size
+  end
+
+  def test_inheriting_copies_exceptions
+    assert_equal RetryDefaultsJob.retry_criteria_checks, InheritTestJob.retry_criteria_checks
+  end
+
+  def test_inheriting_adds_exceptions
+    assert_equal 1, InheritTestWithExtraJob.retry_criteria_checks.size
+  end
+
+  def test_extending_with_resque_retry_doesnt_override_previously_defined_inherited_hook
+    klass = InheritOrderingJobExtendLastSubclass
+    assert_equal 1, klass.retry_criteria_checks.size
+    assert_equal 'test', klass.test_value
+  end
+
+  def test_extending_with_resque_retry_then_defining_inherited_does_not_override_previous_hook
+    klass = InheritOrderingJobExtendFirstSubclass
+    assert_equal 1, klass.retry_criteria_checks.size
+    assert_equal 'test', klass.test_value
+  end
+end

data/test/retry_test.rb

@@ -0,0 +1,173 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class RetryTest < Test::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:testing)
+    @worker.register_worker
+  end
+
+  def test_resque_plugin_lint
+    assert_nothing_raised do
+      Resque::Plugin.lint(Resque::Plugins::Retry)
+    end
+  end
+
+  def test_default_settings
+    assert_equal 1, RetryDefaultsJob.retry_limit, 'default retry limit'
+    assert_equal 0, RetryDefaultsJob.retry_attempt, 'default number of retry attempts'
+    assert_equal nil, RetryDefaultsJob.retry_exceptions, 'default retry exceptions; nil = any'
+    assert_equal 0, RetryDefaultsJob.retry_delay, 'default seconds until retry'
+  end
+
+  def test_retry_once_by_default
+    Resque.enqueue(RetryDefaultsJob)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+  end
+
+  def test_module_retry_defaults
+    Resque.enqueue(RetryModuleDefaultsJob)
+    3.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+  end
+
+  def test_job_args_are_maintained
+    test_args = ['maiow', 'cat', [42, 84]]
+
+    Resque.enqueue(RetryDefaultsJob, *test_args)
+    perform_next_job(@worker)
+
+    assert job = Resque.pop(:testing)
+    assert_equal test_args, job['args']
+  end
+
+  def test_job_args_may_be_modified
+    Resque.enqueue(RetryWithModifiedArgsJob, 'foo', 'bar')
+    perform_next_job(@worker)
+
+    assert job = Resque.pop(:testing)
+    assert_equal ['foobar', 'barbar'], job['args']
+  end
+
+  def test_retry_never_give_up
+    Resque.enqueue(NeverGiveUpJob)
+    10.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 1, Resque.info[:pending], 'pending jobs'
+    assert_equal 10, Resque.info[:failed], 'failed jobs'
+    assert_equal 10, Resque.info[:processed], 'processed job'
+  end
+
+  def test_fail_five_times_then_succeed
+    Resque.enqueue(FailFiveTimesJob)
+    7.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 5, Resque.info[:failed], 'failed jobs'
+    assert_equal 6, Resque.info[:processed], 'processed job'
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+  end
+
+  def test_can_determine_if_exception_may_be_retried
+    assert_equal true, RetryDefaultsJob.retry_exception?(StandardError), 'StandardError may retry'
+    assert_equal true, RetryDefaultsJob.retry_exception?(CustomException), 'CustomException may retry'
+    assert_equal true, RetryDefaultsJob.retry_exception?(HierarchyCustomException), 'HierarchyCustomException may retry'
+
+    assert_equal true, RetryCustomExceptionsJob.retry_exception?(CustomException), 'CustomException may retry'
+    assert_equal true, RetryCustomExceptionsJob.retry_exception?(HierarchyCustomException), 'HierarchyCustomException may retry'
+    assert_equal false, RetryCustomExceptionsJob.retry_exception?(AnotherCustomException), 'AnotherCustomException may not retry'
+  end
+
+  def test_retry_if_failed_and_exception_may_retry
+    Resque.enqueue(RetryCustomExceptionsJob, CustomException)
+    Resque.enqueue(RetryCustomExceptionsJob, HierarchyCustomException)
+    4.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 4, Resque.info[:failed], 'failed jobs'
+    assert_equal 4, Resque.info[:processed], 'processed job'
+    assert_equal 2, Resque.info[:pending], 'pending jobs'
+  end
+
+  def test_do_not_retry_if_failed_and_exception_does_not_allow_retry
+    Resque.enqueue(RetryCustomExceptionsJob, AnotherCustomException)
+    Resque.enqueue(RetryCustomExceptionsJob, RuntimeError)
+    4.times do
+      perform_next_job(@worker)
+    end
+
+    assert_equal 2, Resque.info[:failed], 'failed jobs'
+    assert_equal 2, Resque.info[:processed], 'processed job'
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+  end
+
+  def test_retry_failed_jobs_in_separate_queue
+    Resque.enqueue(JobWithRetryQueue, 'arg1')
+
+    perform_next_job(@worker)
+
+    assert job_from_retry_queue = Resque.pop(:testing_retry)
+    assert_equal ['arg1'], job_from_retry_queue['args']
+  end
+
+  def test_retry_failed_jobs_in_separate_queue
+    Resque.enqueue(DelayedJobWithRetryQueue, 'arg1')
+    Resque.expects(:enqueue_in).with(1, JobRetryQueue, 'arg1')
+
+    perform_next_job(@worker)
+  end
+
+  def test_delete_redis_key_when_job_is_successful
+    Resque.enqueue(GoodJob, 'arg1')
+
+    assert_equal nil, Resque.redis.get(GoodJob.redis_retry_key('arg1'))
+    perform_next_job(@worker)
+    assert_equal nil, Resque.redis.get(GoodJob.redis_retry_key('arg1'))
+  end
+
+  def test_delete_redis_key_after_final_failed_retry
+    Resque.enqueue(FailFiveTimesJob, 'yarrrr')
+    assert_equal nil, Resque.redis.get(FailFiveTimesJob.redis_retry_key('yarrrr'))
+
+    perform_next_job(@worker)
+    assert_equal '0', Resque.redis.get(FailFiveTimesJob.redis_retry_key('yarrrr'))
+
+    perform_next_job(@worker)
+    assert_equal '1', Resque.redis.get(FailFiveTimesJob.redis_retry_key('yarrrr'))
+
+    5.times do
+      perform_next_job(@worker)
+    end
+    assert_equal nil, Resque.redis.get(FailFiveTimesJob.redis_retry_key('yarrrr'))
+
+    assert_equal 5, Resque.info[:failed], 'failed jobs'
+    assert_equal 6, Resque.info[:processed], 'processed job'
+    assert_equal 0, Resque.info[:pending], 'pending jobs'
+  end
+
+  def test_job_without_args_has_no_ending_colon_in_redis_key
+    assert_equal 'resque-retry:GoodJob:yarrrr', GoodJob.redis_retry_key('yarrrr')
+    assert_equal 'resque-retry:GoodJob:foo', GoodJob.redis_retry_key('foo')
+    assert_equal 'resque-retry:GoodJob', GoodJob.redis_retry_key
+  end
+
+  def test_redis_retry_key_removes_whitespace
+    assert_equal 'resque-retry:GoodJob:arg1-removespace', GoodJob.redis_retry_key('arg1', 'remove space')
+  end
+
+end