resque-retry 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY.md +6 -0
- data/README.md +35 -8
- data/lib/resque/plugins/retry.rb +21 -7
- data/test/exponential_backoff_test.rb +21 -20
- data/test/multiple_failure_test.rb +1 -1
- data/test/resque_test.rb +1 -1
- data/test/retry_criteria_test.rb +1 -1
- data/test/retry_exception_delay_test.rb +47 -0
- data/test/retry_inheriting_checks_test.rb +1 -1
- data/test/retry_test.rb +12 -5
- data/test/server_test.rb +1 -1
- data/test/test_helper.rb +1 -0
- data/test/test_jobs.rb +32 -0
- metadata +19 -18
data/HISTORY.md
CHANGED
data/README.md
CHANGED
@@ -155,8 +155,9 @@ more special.
|
|
155
155
|
|
156
156
|
### Sleep After Requeuing
|
157
157
|
|
158
|
-
Sometimes it is useful to delay the worker that failed a job attempt, but
|
159
|
-
by other workers. This can be
|
158
|
+
Sometimes it is useful to delay the worker that failed a job attempt, but
|
159
|
+
still requeue the job for immediate processing by other workers. This can be
|
160
|
+
done with `@sleep_after_requeue`:
|
160
161
|
|
161
162
|
class DeliverWebHook
|
162
163
|
extend Resque::Plugins::Retry
|
@@ -169,14 +170,18 @@ by other workers. This can be done with `@sleep_after_requeue`:
|
|
169
170
|
end
|
170
171
|
end
|
171
172
|
|
172
|
-
This retries the job once and causes the worker that failed to sleep for 5
|
173
|
-
|
174
|
-
|
175
|
-
|
173
|
+
This retries the job once and causes the worker that failed to sleep for 5
|
174
|
+
seconds after requeuing the job. If there are multiple workers in the system
|
175
|
+
this allows the job to be retried immediately while the original worker heals
|
176
|
+
itself.For example failed jobs may cause other (non-worker) OS processes to
|
177
|
+
die. A system monitor such as [god][god] can fix the server while the job is
|
178
|
+
being retried on a different worker.
|
176
179
|
|
177
|
-
`@sleep_after_requeue` is independent of `@retry_delay`.
|
180
|
+
`@sleep_after_requeue` is independent of `@retry_delay`. If you set both, they
|
181
|
+
both take effect.
|
178
182
|
|
179
|
-
You can override the method `sleep_after_requeue` to set the sleep value
|
183
|
+
You can override the method `sleep_after_requeue` to set the sleep value
|
184
|
+
dynamically.
|
180
185
|
|
181
186
|
### Exponential Backoff
|
182
187
|
|
@@ -223,6 +228,28 @@ it so only specific exceptions are retried using `retry_exceptions`:
|
|
223
228
|
The above modification will **only** retry if a `NetworkError` (or subclass)
|
224
229
|
exception is thrown.
|
225
230
|
|
231
|
+
You may also want to specify different retry delays for different exception
|
232
|
+
types. You may optionally set `@retry_exceptions` to a hash where the keys are
|
233
|
+
your specific exception classes to retry on, and the values are your retry
|
234
|
+
delays in seconds or an array of retry delays to be used similar to
|
235
|
+
exponential backoff.
|
236
|
+
|
237
|
+
class DeliverSMS
|
238
|
+
extend Resque::Plugins::Retry
|
239
|
+
@queue = :mt_messages
|
240
|
+
|
241
|
+
@retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }
|
242
|
+
|
243
|
+
def self.perform(mt_id, mobile_number, message)
|
244
|
+
heavy_lifting
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
In the above example, Resque would retry any `DeliverSMS` jobs which throw a
|
249
|
+
`NetworkError` or `SystemCallError`. If the job throws a `NetworkError` it
|
250
|
+
will be retried 30 seconds later, if it throws `SystemCallError` it will first
|
251
|
+
retry 120 seconds later then subsequent retry attempts 240 seconds later.
|
252
|
+
|
226
253
|
### Custom Retry Criteria Check Callbacks
|
227
254
|
|
228
255
|
You may define custom retry criteria callbacks:
|
data/lib/resque/plugins/retry.rb
CHANGED
@@ -86,8 +86,14 @@ module Resque
|
|
86
86
|
# Number of seconds to delay until the job is retried.
|
87
87
|
#
|
88
88
|
# @return [Number] number of seconds to delay
|
89
|
-
def retry_delay
|
90
|
-
@
|
89
|
+
def retry_delay(exception_class = nil)
|
90
|
+
if @retry_exceptions.is_a?(Hash)
|
91
|
+
delay = @retry_exceptions[exception_class] || 0
|
92
|
+
# allow an array of delays.
|
93
|
+
delay.is_a?(Array) ? delay[retry_attempt] || delay.last : delay
|
94
|
+
else
|
95
|
+
@retry_delay ||= 0
|
96
|
+
end
|
91
97
|
end
|
92
98
|
|
93
99
|
# @abstract
|
@@ -122,7 +128,11 @@ module Resque
|
|
122
128
|
#
|
123
129
|
# @return [Array, nil]
|
124
130
|
def retry_exceptions
|
125
|
-
@retry_exceptions
|
131
|
+
if @retry_exceptions.is_a?(Hash)
|
132
|
+
@retry_exceptions.keys
|
133
|
+
else
|
134
|
+
@retry_exceptions ||= nil
|
135
|
+
end
|
126
136
|
end
|
127
137
|
|
128
138
|
# Test if the retry criteria is valid.
|
@@ -191,12 +201,16 @@ module Resque
|
|
191
201
|
end
|
192
202
|
|
193
203
|
# Retries the job.
|
194
|
-
def try_again(*args)
|
195
|
-
|
204
|
+
def try_again(exception, *args)
|
205
|
+
# some plugins define retry_delay and have it take no arguments, so rather than break those,
|
206
|
+
# we'll just check here to see whether it takes the additional exception class argument or not
|
207
|
+
temp_retry_delay = ([-1, 1].include?(method(:retry_delay).arity) ? retry_delay(exception.class) : retry_delay)
|
208
|
+
|
209
|
+
if temp_retry_delay <= 0
|
196
210
|
# If the delay is 0, no point passing it through the scheduler
|
197
211
|
Resque.enqueue(self, *args_for_retry(*args))
|
198
212
|
else
|
199
|
-
Resque.enqueue_in(
|
213
|
+
Resque.enqueue_in(temp_retry_delay, self, *args_for_retry(*args))
|
200
214
|
end
|
201
215
|
sleep(sleep_after_requeue) if sleep_after_requeue > 0
|
202
216
|
end
|
@@ -223,7 +237,7 @@ module Resque
|
|
223
237
|
# Otherwise the retry attempt count is deleted from Redis.
|
224
238
|
def on_failure_retry(exception, *args)
|
225
239
|
if retry_criteria_valid?(exception, *args)
|
226
|
-
try_again(*args)
|
240
|
+
try_again(exception, *args)
|
227
241
|
else
|
228
242
|
Resque.redis.del(redis_retry_key(*args))
|
229
243
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/test_helper'
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/test_helper')
|
2
2
|
|
3
3
|
class ExponentialBackoffTest < MiniTest::Unit::TestCase
|
4
4
|
def setup
|
@@ -13,21 +13,21 @@ class ExponentialBackoffTest < MiniTest::Unit::TestCase
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def test_default_backoff_strategy
|
16
|
-
|
16
|
+
start_time = Time.now.to_i
|
17
17
|
Resque.enqueue(ExponentialBackoffJob)
|
18
18
|
|
19
19
|
perform_next_job @worker
|
20
|
-
assert_equal 1, Resque.info[:processed],
|
21
|
-
assert_equal 1, Resque.info[:failed],
|
22
|
-
assert_equal 1, Resque.info[:pending],
|
20
|
+
assert_equal 1, Resque.info[:processed], '1 processed job'
|
21
|
+
assert_equal 1, Resque.info[:failed], 'first ever run, and it should have failed, but never retried'
|
22
|
+
assert_equal 1, Resque.info[:pending], '1 pending job, because it never hits the scheduler'
|
23
23
|
|
24
24
|
perform_next_job @worker
|
25
|
-
assert_equal 2, Resque.info[:processed],
|
26
|
-
assert_equal 2, Resque.info[:failed],
|
27
|
-
assert_equal 0, Resque.info[:pending],
|
25
|
+
assert_equal 2, Resque.info[:processed], '2nd run, but first retry'
|
26
|
+
assert_equal 2, Resque.info[:failed], 'should of failed again, this is the first retry attempt'
|
27
|
+
assert_equal 0, Resque.info[:pending], '0 pending jobs, it should be in the delayed queue'
|
28
28
|
|
29
29
|
delayed = Resque.delayed_queue_peek(0, 1)
|
30
|
-
|
30
|
+
assert_in_delta (start_time + 60), delayed[0], 1.00, '2nd delay' # the first had a zero delay.
|
31
31
|
|
32
32
|
5.times do
|
33
33
|
Resque.enqueue(ExponentialBackoffJob)
|
@@ -35,27 +35,28 @@ class ExponentialBackoffTest < MiniTest::Unit::TestCase
|
|
35
35
|
end
|
36
36
|
|
37
37
|
delayed = Resque.delayed_queue_peek(0, 5)
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
38
|
+
assert_in_delta (start_time + 600), delayed[1], 1.00, '3rd delay'
|
39
|
+
assert_in_delta (start_time + 3600), delayed[2], 1.00, '4th delay'
|
40
|
+
assert_in_delta (start_time + 10_800), delayed[3], 1.00, '5th delay'
|
41
|
+
assert_in_delta (start_time + 21_600), delayed[4], 1.00, '6th delay'
|
42
42
|
end
|
43
43
|
|
44
44
|
def test_custom_backoff_strategy
|
45
|
-
|
45
|
+
start_time = Time.now.to_i
|
46
46
|
4.times do
|
47
47
|
Resque.enqueue(CustomExponentialBackoffJob, 'http://lividpenguin.com', 1305, 'cd8079192d379dc612f17c660591a6cfb05f1dda')
|
48
48
|
perform_next_job @worker
|
49
49
|
end
|
50
50
|
|
51
51
|
delayed = Resque.delayed_queue_peek(0, 3)
|
52
|
-
|
53
|
-
|
54
|
-
|
52
|
+
assert_in_delta (start_time + 10), delayed[0], 1.00, '1st delay'
|
53
|
+
assert_in_delta (start_time + 20), delayed[1], 1.00, '2nd delay'
|
54
|
+
assert_in_delta (start_time + 30), delayed[2], 1.00, '3rd delay'
|
55
|
+
|
55
56
|
assert_equal 2, Resque.delayed_timestamp_size(delayed[2]), '4th delay should share delay with 3rd'
|
56
57
|
|
57
|
-
assert_equal 4, Resque.info[:processed],
|
58
|
-
assert_equal 4, Resque.info[:failed],
|
59
|
-
assert_equal 0, Resque.info[:pending],
|
58
|
+
assert_equal 4, Resque.info[:processed], 'processed jobs'
|
59
|
+
assert_equal 4, Resque.info[:failed], 'failed jobs'
|
60
|
+
assert_equal 0, Resque.info[:pending], 'pending jobs'
|
60
61
|
end
|
61
62
|
end
|
data/test/resque_test.rb
CHANGED
data/test/retry_criteria_test.rb
CHANGED
@@ -0,0 +1,47 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/test_helper')
|
2
|
+
|
3
|
+
class RetryTest < MiniTest::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Resque.redis.flushall
|
6
|
+
@worker = Resque::Worker.new(:testing)
|
7
|
+
@worker.register_worker
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_retry_delay_per_exception_single_delay
|
11
|
+
# store start time for later comparison with retry delay.
|
12
|
+
start_time = Time.now.to_i
|
13
|
+
|
14
|
+
# work the job a couple of times to build up some delayed jobs.
|
15
|
+
3.times do
|
16
|
+
Resque.enqueue(PerExceptionClassRetryCountJob)
|
17
|
+
perform_next_job(@worker)
|
18
|
+
end
|
19
|
+
|
20
|
+
# double check job counts.
|
21
|
+
assert_equal 3, Resque.info[:failed], 'failed jobs'
|
22
|
+
assert_equal 3, Resque.info[:processed], 'processed job'
|
23
|
+
assert_equal 0, Resque.info[:pending], 'zero pending jobs as their delayed'
|
24
|
+
|
25
|
+
# now lets see if the delays are correct?
|
26
|
+
delayed = Resque.delayed_queue_peek(0, 3)
|
27
|
+
assert_in_delta (start_time + 7), delayed[0], 1.00, 'retry delay timestamp'
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_retry_delay_per_exception_multiple_delay
|
31
|
+
# store start time for later comparison with retry delay.
|
32
|
+
start_time = Time.now.to_i
|
33
|
+
|
34
|
+
# work the job a couple of times to build up some delayed jobs.
|
35
|
+
3.times do
|
36
|
+
Resque.enqueue(PerExceptionClassRetryCountArrayJob)
|
37
|
+
perform_next_job(@worker)
|
38
|
+
end
|
39
|
+
|
40
|
+
# now lets see if the delays are correct?
|
41
|
+
delayed = Resque.delayed_queue_peek(0, 3)
|
42
|
+
assert_in_delta (start_time + 5), delayed[0], 1.00, '1st retry delay timestamp'
|
43
|
+
assert_in_delta (start_time + 10), delayed[1], 1.00, '2nd retry delay timestamp'
|
44
|
+
assert_in_delta (start_time + 15), delayed[2], 1.00, '3rd retry delay timestamp'
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
data/test/retry_test.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/test_helper'
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/test_helper')
|
2
2
|
|
3
3
|
class RetryTest < MiniTest::Unit::TestCase
|
4
4
|
def setup
|
@@ -96,18 +96,18 @@ class RetryTest < MiniTest::Unit::TestCase
|
|
96
96
|
assert_equal 0, Resque.info[:failed], 'failed jobs'
|
97
97
|
Resque.enqueue(SleepDelay1SecondJob)
|
98
98
|
before = Time.now
|
99
|
-
|
99
|
+
2.times do
|
100
100
|
perform_next_job(@worker)
|
101
101
|
end
|
102
102
|
actual_delay = Time.now - before
|
103
|
-
|
103
|
+
|
104
104
|
assert actual_delay >= 1, "did not sleep long enough: #{actual_delay} seconds"
|
105
105
|
assert actual_delay < 2, "slept too long: #{actual_delay} seconds"
|
106
106
|
assert_equal 1, Resque.info[:failed], 'failed jobs'
|
107
107
|
assert_equal 2, Resque.info[:processed], 'processed job'
|
108
108
|
assert_equal 0, Resque.info[:pending], 'pending jobs'
|
109
109
|
end
|
110
|
-
|
110
|
+
|
111
111
|
def test_can_determine_if_exception_may_be_retried
|
112
112
|
assert_equal true, RetryDefaultsJob.retry_exception?(StandardError), 'StandardError may retry'
|
113
113
|
assert_equal true, RetryDefaultsJob.retry_exception?(CustomException), 'CustomException may retry'
|
@@ -180,4 +180,11 @@ class RetryTest < MiniTest::Unit::TestCase
|
|
180
180
|
assert_equal 'resque-retry:GoodJob:arg1-removespace', GoodJob.redis_retry_key('arg1', 'remove space')
|
181
181
|
end
|
182
182
|
|
183
|
-
|
183
|
+
def test_retry_delay
|
184
|
+
assert_equal 3, NormalRetryCountJob.retry_delay
|
185
|
+
assert_equal 7, PerExceptionClassRetryCountJob.retry_delay(RuntimeError)
|
186
|
+
assert_equal 11, PerExceptionClassRetryCountJob.retry_delay(Exception)
|
187
|
+
assert_equal 13, PerExceptionClassRetryCountJob.retry_delay(Timeout::Error)
|
188
|
+
end
|
189
|
+
|
190
|
+
end
|
data/test/server_test.rb
CHANGED
data/test/test_helper.rb
CHANGED
data/test/test_jobs.rb
CHANGED
@@ -269,3 +269,35 @@ class CustomIdentifierFailingJob
|
|
269
269
|
raise 'failed'
|
270
270
|
end
|
271
271
|
end
|
272
|
+
|
273
|
+
class NormalRetryCountJob
|
274
|
+
extend Resque::Plugins::Retry
|
275
|
+
|
276
|
+
@queue = :testing
|
277
|
+
@retry_delay = 3
|
278
|
+
@retry_exceptions = [RuntimeError, Exception, Timeout::Error]
|
279
|
+
end
|
280
|
+
|
281
|
+
class PerExceptionClassRetryCountJob
|
282
|
+
extend Resque::Plugins::Retry
|
283
|
+
|
284
|
+
@queue = :testing
|
285
|
+
@retry_limit = 3
|
286
|
+
@retry_exceptions = { RuntimeError => 7, Exception => 11, Timeout::Error => 13 }
|
287
|
+
|
288
|
+
def self.perform
|
289
|
+
raise RuntimeError, 'I always fail with a RuntimeError'
|
290
|
+
end
|
291
|
+
end
|
292
|
+
|
293
|
+
class PerExceptionClassRetryCountArrayJob
|
294
|
+
extend Resque::Plugins::Retry
|
295
|
+
|
296
|
+
@queue = :testing
|
297
|
+
@retry_limit = 3
|
298
|
+
@retry_exceptions = { Exception => 11, RuntimeError => [5, 10, 15], Timeout::Error => [2, 4, 6, 8, 10] }
|
299
|
+
|
300
|
+
def self.perform
|
301
|
+
raise RuntimeError, 'I always fail with a RuntimeError'
|
302
|
+
end
|
303
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-retry
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,11 +10,11 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2011-
|
13
|
+
date: 2011-12-08 00:00:00.000000000Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
17
|
-
requirement: &
|
17
|
+
requirement: &2153685420 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
@@ -22,10 +22,10 @@ dependencies:
|
|
22
22
|
version: '0'
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *2153685420
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
27
|
name: resque
|
28
|
-
requirement: &
|
28
|
+
requirement: &2153679300 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
31
|
- - ! '>='
|
@@ -33,10 +33,10 @@ dependencies:
|
|
33
33
|
version: 1.8.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
|
-
version_requirements: *
|
36
|
+
version_requirements: *2153679300
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
38
|
name: resque-scheduler
|
39
|
-
requirement: &
|
39
|
+
requirement: &2153678300 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ! '>='
|
@@ -44,10 +44,10 @@ dependencies:
|
|
44
44
|
version: 1.8.0
|
45
45
|
type: :runtime
|
46
46
|
prerelease: false
|
47
|
-
version_requirements: *
|
47
|
+
version_requirements: *2153678300
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: minitest
|
50
|
-
requirement: &
|
50
|
+
requirement: &2153677800 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
53
|
- - ! '>='
|
@@ -55,10 +55,10 @@ dependencies:
|
|
55
55
|
version: '0'
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
|
-
version_requirements: *
|
58
|
+
version_requirements: *2153677800
|
59
59
|
- !ruby/object:Gem::Dependency
|
60
60
|
name: rack-test
|
61
|
-
requirement: &
|
61
|
+
requirement: &2153660820 !ruby/object:Gem::Requirement
|
62
62
|
none: false
|
63
63
|
requirements:
|
64
64
|
- - ! '>='
|
@@ -66,10 +66,10 @@ dependencies:
|
|
66
66
|
version: '0'
|
67
67
|
type: :development
|
68
68
|
prerelease: false
|
69
|
-
version_requirements: *
|
69
|
+
version_requirements: *2153660820
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
71
|
name: yard
|
72
|
-
requirement: &
|
72
|
+
requirement: &2153659700 !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
75
|
- - ! '>='
|
@@ -77,10 +77,10 @@ dependencies:
|
|
77
77
|
version: '0'
|
78
78
|
type: :development
|
79
79
|
prerelease: false
|
80
|
-
version_requirements: *
|
80
|
+
version_requirements: *2153659700
|
81
81
|
- !ruby/object:Gem::Dependency
|
82
82
|
name: json
|
83
|
-
requirement: &
|
83
|
+
requirement: &2153659000 !ruby/object:Gem::Requirement
|
84
84
|
none: false
|
85
85
|
requirements:
|
86
86
|
- - ! '>='
|
@@ -88,10 +88,10 @@ dependencies:
|
|
88
88
|
version: '0'
|
89
89
|
type: :development
|
90
90
|
prerelease: false
|
91
|
-
version_requirements: *
|
91
|
+
version_requirements: *2153659000
|
92
92
|
- !ruby/object:Gem::Dependency
|
93
93
|
name: simplecov
|
94
|
-
requirement: &
|
94
|
+
requirement: &2153658420 !ruby/object:Gem::Requirement
|
95
95
|
none: false
|
96
96
|
requirements:
|
97
97
|
- - ! '>='
|
@@ -99,7 +99,7 @@ dependencies:
|
|
99
99
|
version: 0.3.0
|
100
100
|
type: :development
|
101
101
|
prerelease: false
|
102
|
-
version_requirements: *
|
102
|
+
version_requirements: *2153658420
|
103
103
|
description: ! " resque-retry provides retry, delay and exponential backoff support
|
104
104
|
for\n resque jobs.\n\n Features:\n\n * Redis backed retry count/limit.\n * Retry
|
105
105
|
on all or specific exceptions.\n * Exponential backoff (varying the delay between
|
@@ -119,6 +119,7 @@ files:
|
|
119
119
|
- test/redis-test.conf
|
120
120
|
- test/resque_test.rb
|
121
121
|
- test/retry_criteria_test.rb
|
122
|
+
- test/retry_exception_delay_test.rb
|
122
123
|
- test/retry_inheriting_checks_test.rb
|
123
124
|
- test/retry_test.rb
|
124
125
|
- test/server_test.rb
|