resque-retry 1.5.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +24 -0
- data/.gitignore +2 -0
- data/HISTORY.md +115 -21
- data/README.md +158 -21
- data/lib/resque/failure/multiple_with_retry_suppression.rb +77 -24
- data/lib/resque/plugins/exponential_backoff.rb +10 -5
- data/lib/resque/plugins/retry.rb +93 -24
- data/lib/resque-retry/server.rb +13 -5
- data/lib/resque-retry/version.rb +1 -1
- data/resque-retry.gemspec +3 -4
- data/test/exponential_backoff_test.rb +7 -0
- data/test/ignore_exceptions_test.rb +26 -0
- data/test/multiple_failure_test.rb +41 -8
- data/test/resque_inline_test.rb +23 -0
- data/test/resque_test.rb +18 -4
- data/test/retry_custom_delay_test.rb +23 -0
- data/test/retry_exception_delay_test.rb +9 -1
- data/test/retry_queue_test.rb +25 -0
- data/test/retry_test.rb +23 -27
- data/test/test_helper.rb +31 -25
- data/test/test_jobs.rb +103 -13
- metadata +29 -15
- data/.travis.yml +0 -15
@@ -3,10 +3,9 @@ require 'resque/plugins/retry/logging'
|
|
3
3
|
|
4
4
|
module Resque
|
5
5
|
module Failure
|
6
|
-
|
7
6
|
# A multiple failure backend, with retry suppression
|
8
7
|
#
|
9
|
-
# For example: if you had a job that could retry 5 times, your failure
|
8
|
+
# For example: if you had a job that could retry 5 times, your failure
|
10
9
|
# backends are not notified unless the _final_ retry attempt also fails.
|
11
10
|
#
|
12
11
|
# Example:
|
@@ -28,17 +27,44 @@ module Resque
|
|
28
27
|
#
|
29
28
|
# @api private
|
30
29
|
def save
|
31
|
-
|
30
|
+
args = args_from(payload)
|
31
|
+
|
32
|
+
log_message 'failure backend save', args, exception
|
32
33
|
|
33
34
|
retryable = retryable?
|
34
35
|
job_being_retried = retryable && retrying?
|
35
36
|
|
36
37
|
if !job_being_retried
|
37
|
-
log_message
|
38
|
+
log_message(
|
39
|
+
"#{retryable ? '' : 'non-'}retryable job is not being retried - sending failure to superclass",
|
40
|
+
args,
|
41
|
+
exception
|
42
|
+
)
|
43
|
+
|
38
44
|
cleanup_retry_failure_log!
|
39
|
-
super
|
40
|
-
|
41
|
-
|
45
|
+
return super
|
46
|
+
end
|
47
|
+
|
48
|
+
# some plugins define retry_delay and have it take no arguments, so rather than break those,
|
49
|
+
# we'll just check here to see whether it takes the additional exception class argument or not
|
50
|
+
# we also allow all job args to be passed to a custom `retry_delay` method
|
51
|
+
retry_delay_arity = klass.method(:retry_delay).arity
|
52
|
+
|
53
|
+
calculated_retry_delay = if [-2, 2].include?(retry_delay_arity)
|
54
|
+
klass.retry_delay(exception.class, *args)
|
55
|
+
elsif [-1, 1].include?(retry_delay_arity)
|
56
|
+
klass.retry_delay(exception.class)
|
57
|
+
else
|
58
|
+
klass.retry_delay
|
59
|
+
end
|
60
|
+
|
61
|
+
if calculated_retry_delay > 0
|
62
|
+
log_message(
|
63
|
+
"retry_delay: #{calculated_retry_delay} > 0 - saving details in Redis",
|
64
|
+
args,
|
65
|
+
exception
|
66
|
+
)
|
67
|
+
|
42
68
|
data = {
|
43
69
|
:failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
|
44
70
|
:payload => payload,
|
@@ -48,10 +74,19 @@ module Resque
|
|
48
74
|
:worker => worker.to_s,
|
49
75
|
:queue => queue
|
50
76
|
}
|
77
|
+
data = Resque.encode(data)
|
51
78
|
|
52
|
-
Resque.redis.setex(
|
79
|
+
Resque.redis.setex(
|
80
|
+
failure_key,
|
81
|
+
2 * calculated_retry_delay,
|
82
|
+
data
|
83
|
+
)
|
53
84
|
else
|
54
|
-
log_message
|
85
|
+
log_message(
|
86
|
+
"retry_delay: #{calculated_retry_delay} <= 0 - ignoring",
|
87
|
+
args,
|
88
|
+
exception
|
89
|
+
)
|
55
90
|
end
|
56
91
|
end
|
57
92
|
|
@@ -62,25 +97,45 @@ module Resque
|
|
62
97
|
'failure-' + retry_key
|
63
98
|
end
|
64
99
|
|
65
|
-
|
66
|
-
|
67
|
-
#
|
68
|
-
|
69
|
-
|
100
|
+
# Monkey-patch this in for now since it is a hard requirement for the
|
101
|
+
# "retry-all" functionality to work via the Web UI.
|
102
|
+
#
|
103
|
+
# This can be removed when the following PR has been merged into `Resque`
|
104
|
+
# itself:
|
105
|
+
#
|
106
|
+
# https://github.com/resque/resque/pull/1659
|
107
|
+
#
|
108
|
+
# @api public
|
109
|
+
class_eval do |klass|
|
110
|
+
if !klass.respond_to?(:requeue_queue)
|
111
|
+
def klass.requeue_queue(queue)
|
112
|
+
classes.first.requeue_queue(queue)
|
113
|
+
end
|
114
|
+
end
|
70
115
|
end
|
71
116
|
|
72
|
-
|
73
|
-
|
117
|
+
protected
|
118
|
+
|
119
|
+
def args_from(payload)
|
120
|
+
(payload || {})['args'].dup
|
74
121
|
end
|
75
122
|
|
76
|
-
def
|
77
|
-
|
123
|
+
def cleanup_retry_failure_log!
|
124
|
+
Resque.redis.del(failure_key) if retryable?
|
78
125
|
end
|
79
126
|
|
80
127
|
def failure_key
|
81
128
|
self.class.failure_key(retry_key)
|
82
129
|
end
|
83
130
|
|
131
|
+
def klass
|
132
|
+
Resque::Job.new(nil, nil).constantize(payload['class'])
|
133
|
+
end
|
134
|
+
|
135
|
+
def retry_key
|
136
|
+
klass.redis_retry_key(*payload['args'])
|
137
|
+
end
|
138
|
+
|
84
139
|
def retryable?
|
85
140
|
klass.respond_to?(:redis_retry_key)
|
86
141
|
rescue NameError
|
@@ -88,15 +143,13 @@ module Resque
|
|
88
143
|
end
|
89
144
|
|
90
145
|
def retrying?
|
91
|
-
|
146
|
+
redis_key_exists?(retry_key)
|
92
147
|
end
|
93
148
|
|
94
|
-
|
95
|
-
Resque.redis.del(failure_key) if retryable?
|
96
|
-
end
|
149
|
+
private
|
97
150
|
|
98
|
-
def
|
99
|
-
(
|
151
|
+
def redis_key_exists?(key)
|
152
|
+
![false, 0].include?(Resque.redis.exists(key) || false)
|
100
153
|
end
|
101
154
|
end
|
102
155
|
end
|
@@ -52,12 +52,16 @@ module Resque
|
|
52
52
|
#
|
53
53
|
# @api private
|
54
54
|
def self.extended(receiver)
|
55
|
+
retry_delay_multiplicand_min = DEFAULT_RETRY_DELAY_MULTIPLICAND_MIN
|
55
56
|
retry_delay_multiplicand_min = \
|
56
|
-
receiver.instance_variable_get(
|
57
|
-
|
57
|
+
receiver.instance_variable_get(:@retry_delay_multiplicand_min) \
|
58
|
+
if receiver.instance_variable_defined?(:@retry_delay_multiplicand_min)
|
59
|
+
|
60
|
+
retry_delay_multiplicand_max = DEFAULT_RETRY_DELAY_MULTIPLICAND_MAX
|
58
61
|
retry_delay_multiplicand_max = \
|
59
|
-
receiver.instance_variable_get(
|
60
|
-
|
62
|
+
receiver.instance_variable_get(:@retry_delay_multiplicand_max) \
|
63
|
+
if receiver.instance_variable_defined?(:@retry_delay_multiplicand_max)
|
64
|
+
|
61
65
|
if retry_delay_multiplicand_min > retry_delay_multiplicand_max
|
62
66
|
raise InvalidRetryDelayMultiplicandConfigurationException.new(
|
63
67
|
%{"@retry_delay_multiplicand_min" must be less than or equal to "@retry_delay_multiplicand_max"}
|
@@ -76,10 +80,11 @@ module Resque
|
|
76
80
|
|
77
81
|
# Selects the delay from the backoff strategy
|
78
82
|
#
|
83
|
+
# @param _ [Exception] unused exception argument for signature parity
|
79
84
|
# @return [Number] seconds to delay until the next retry.
|
80
85
|
#
|
81
86
|
# @api private
|
82
|
-
def retry_delay
|
87
|
+
def retry_delay(_ = nil)
|
83
88
|
delay = backoff_strategy[retry_attempt] || backoff_strategy.last
|
84
89
|
# if the values are the same don't bother generating a random number, if
|
85
90
|
# the delta is zero, some platforms will raise an error
|
data/lib/resque/plugins/retry.rb
CHANGED
@@ -43,11 +43,24 @@ module Resque
|
|
43
43
|
# @api public
|
44
44
|
class AmbiguousRetryStrategyException < StandardError; end
|
45
45
|
|
46
|
+
# Raised if there is a problem with the configuration of resque-retry.
|
47
|
+
#
|
48
|
+
# @api public
|
49
|
+
class RetryConfigurationException < StandardError; end
|
50
|
+
|
46
51
|
# Fail fast, when extended, if the "receiver" is misconfigured
|
47
52
|
#
|
48
53
|
# @api private
|
49
54
|
def self.extended(receiver)
|
50
|
-
|
55
|
+
retry_exceptions = nil
|
56
|
+
retry_exceptions = receiver.instance_variable_get(:@retry_exceptions) \
|
57
|
+
if receiver.instance_variable_defined?(:@retry_exceptions)
|
58
|
+
|
59
|
+
fatal_exceptions = nil
|
60
|
+
fatal_exceptions = receiver.instance_variable_get(:@fatal_exceptions) \
|
61
|
+
if receiver.instance_variable_defined?(:@fatal_exceptions)
|
62
|
+
|
63
|
+
if fatal_exceptions && retry_exceptions
|
51
64
|
raise AmbiguousRetryStrategyException.new(%{You can't define both "@fatal_exceptions" and "@retry_exceptions"})
|
52
65
|
end
|
53
66
|
end
|
@@ -94,11 +107,27 @@ module Resque
|
|
94
107
|
# A retry limit of 0 will *never* retry.
|
95
108
|
# A retry limit of -1 or below will retry forever.
|
96
109
|
#
|
110
|
+
# The default value is: `1` or in the case of where `@retry_exceptions` is
|
111
|
+
# specified, and it contains one or more `Array` values, the maximum
|
112
|
+
# length will be used (e.g. `@retry_exceptions = { NetworkError => 30, SystemCallError => [120, 240] }`
|
113
|
+
# would return `2` because `SystemCallError` _should_ be attempted at
|
114
|
+
# least twice to respect the specified configuration).
|
115
|
+
#
|
97
116
|
# @return [Fixnum]
|
98
117
|
#
|
99
118
|
# @api public
|
100
119
|
def retry_limit
|
101
|
-
@retry_limit ||=
|
120
|
+
@retry_limit ||= begin
|
121
|
+
default_retry_limit = 1
|
122
|
+
if instance_variable_defined?(:@retry_exceptions) && @retry_exceptions.is_a?(Hash)
|
123
|
+
@retry_exceptions.values.each do |value|
|
124
|
+
if value.is_a?(Array) && value.length > default_retry_limit
|
125
|
+
default_retry_limit = value.length
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
default_retry_limit
|
130
|
+
end
|
102
131
|
end
|
103
132
|
|
104
133
|
# Number of retry attempts used to try and perform the job
|
@@ -115,13 +144,22 @@ module Resque
|
|
115
144
|
|
116
145
|
# @abstract
|
117
146
|
# Number of seconds to delay until the job is retried
|
147
|
+
# If @retry_exceptions is a Hash and there is no delay defined for exception_class,
|
148
|
+
# looks for closest superclass and assigns it's delay to @retry_exceptions[exception_class]
|
118
149
|
#
|
119
150
|
# @return [Number] number of seconds to delay
|
120
151
|
#
|
121
152
|
# @api public
|
122
153
|
def retry_delay(exception_class = nil)
|
123
|
-
if
|
124
|
-
|
154
|
+
if \
|
155
|
+
!exception_class.nil? && \
|
156
|
+
instance_variable_defined?(:@retry_exceptions) && \
|
157
|
+
@retry_exceptions.is_a?(Hash)
|
158
|
+
delay = @retry_exceptions[exception_class] ||= begin
|
159
|
+
relevant_definitions = \
|
160
|
+
@retry_exceptions.select { |ex| exception_class <= ex }
|
161
|
+
relevant_definitions.any? ? relevant_definitions.sort.first[1] : 0
|
162
|
+
end
|
125
163
|
# allow an array of delays.
|
126
164
|
delay.is_a?(Array) ? delay[retry_attempt] || delay.last : delay
|
127
165
|
else
|
@@ -131,7 +169,7 @@ module Resque
|
|
131
169
|
|
132
170
|
# @abstract
|
133
171
|
# Number of seconds to sleep after job is requeued
|
134
|
-
#
|
172
|
+
#
|
135
173
|
# @return [Number] number of seconds to sleep
|
136
174
|
#
|
137
175
|
# @api public
|
@@ -150,6 +188,16 @@ module Resque
|
|
150
188
|
@retry_job_delegate ||= nil
|
151
189
|
end
|
152
190
|
|
191
|
+
# @abstract
|
192
|
+
# Specify the queue that the job should be placed in upon failure
|
193
|
+
#
|
194
|
+
# @return [Symbol] Symbol representing queue that job should be placed in
|
195
|
+
#
|
196
|
+
# @api public
|
197
|
+
def retry_queue(exception, *args)
|
198
|
+
nil
|
199
|
+
end
|
200
|
+
|
153
201
|
# @abstract
|
154
202
|
# Modify the arguments used to retry the job. Use this to do something
|
155
203
|
# other than try the exact same job again
|
@@ -158,15 +206,7 @@ module Resque
|
|
158
206
|
#
|
159
207
|
# @api public
|
160
208
|
def retry_args(*args)
|
161
|
-
|
162
|
-
# invoke it, but warn that it is deprecated (and will be removed in a
|
163
|
-
# future revision)
|
164
|
-
if respond_to?(:args_for_retry)
|
165
|
-
warn "`Resque::Plugins::Retry#args_for_retry` is deprecated, please use `Resque::Plugins::Retry#retry_args` instead."
|
166
|
-
args_for_retry(*args)
|
167
|
-
else
|
168
|
-
args
|
169
|
-
end
|
209
|
+
args.dup
|
170
210
|
end
|
171
211
|
|
172
212
|
# @abstract
|
@@ -238,7 +278,7 @@ module Resque
|
|
238
278
|
#
|
239
279
|
# @api public
|
240
280
|
def retry_exceptions
|
241
|
-
if @retry_exceptions.is_a?(Hash)
|
281
|
+
if instance_variable_defined?(:@retry_exceptions) && @retry_exceptions.is_a?(Hash)
|
242
282
|
@retry_exceptions.keys
|
243
283
|
else
|
244
284
|
@retry_exceptions ||= nil
|
@@ -264,7 +304,7 @@ module Resque
|
|
264
304
|
# if the retry limit was reached, dont bother checking anything else.
|
265
305
|
if retry_limit_reached?
|
266
306
|
log_message 'retry limit reached', args, exception
|
267
|
-
return false
|
307
|
+
return false
|
268
308
|
end
|
269
309
|
|
270
310
|
# We always want to retry if the exception matches.
|
@@ -362,14 +402,27 @@ module Resque
|
|
362
402
|
|
363
403
|
# some plugins define retry_delay and have it take no arguments, so rather than break those,
|
364
404
|
# we'll just check here to see whether it takes the additional exception class argument or not
|
365
|
-
|
405
|
+
# we also allow all job args to be passed to a custom `retry_delay` method
|
406
|
+
retry_delay_arity = method(:retry_delay).arity
|
407
|
+
|
408
|
+
temp_retry_delay = if [-2, 2].include?(retry_delay_arity)
|
409
|
+
retry_delay(exception.class, *args)
|
410
|
+
elsif [-1, 1].include?(retry_delay_arity)
|
411
|
+
retry_delay(exception.class)
|
412
|
+
else
|
413
|
+
retry_delay
|
414
|
+
end
|
415
|
+
|
416
|
+
retry_job_class = retry_job_delegate ? retry_job_delegate : self
|
366
417
|
|
367
|
-
retry_in_queue =
|
368
|
-
|
418
|
+
retry_in_queue = retry_queue(exception, *args)
|
419
|
+
retry_in_queue ||= Resque.queue_from_class(retry_job_class)
|
420
|
+
|
421
|
+
log_message "retry delay: #{temp_retry_delay} for queue: #{retry_in_queue}", args, exception
|
369
422
|
|
370
423
|
# remember that this job is now being retried. before_perform_retry will increment
|
371
424
|
# this so it represents the retry count, and MultipleWithRetrySuppression uses
|
372
|
-
# the existence of this to determine if the job should be sent to the
|
425
|
+
# the existence of this to determine if the job should be sent to the
|
373
426
|
# parent failure backend (e.g. failed queue) or not. Removing this means
|
374
427
|
# jobs that fail before ::perform will be both retried and sent to the failed queue.
|
375
428
|
Resque.redis.setnx(redis_retry_key(*args), -1)
|
@@ -378,9 +431,9 @@ module Resque
|
|
378
431
|
|
379
432
|
if temp_retry_delay <= 0
|
380
433
|
# If the delay is 0, no point passing it through the scheduler
|
381
|
-
Resque.
|
434
|
+
Resque.enqueue_to(retry_in_queue, retry_job_class, *retry_args)
|
382
435
|
else
|
383
|
-
Resque.
|
436
|
+
Resque.enqueue_in_with_queue(retry_in_queue, temp_retry_delay, retry_job_class, *retry_args)
|
384
437
|
end
|
385
438
|
|
386
439
|
# remove retry key from redis if we handed retry off to another queue.
|
@@ -406,6 +459,7 @@ module Resque
|
|
406
459
|
#
|
407
460
|
# @api private
|
408
461
|
def before_perform_retry(*args)
|
462
|
+
return if Resque.inline?
|
409
463
|
log_message 'before_perform_retry', args
|
410
464
|
@on_failure_retry_hook_already_called = false
|
411
465
|
|
@@ -418,7 +472,8 @@ module Resque
|
|
418
472
|
# set/update the "retry_key" expiration
|
419
473
|
if expire_retry_key_after
|
420
474
|
log_message "updating expiration for retry key: #{retry_key}", args
|
421
|
-
|
475
|
+
exception_class = Object.const_get(args[0]) rescue nil
|
476
|
+
Resque.redis.expire(retry_key, retry_delay(exception_class) + expire_retry_key_after)
|
422
477
|
end
|
423
478
|
end
|
424
479
|
|
@@ -428,6 +483,7 @@ module Resque
|
|
428
483
|
#
|
429
484
|
# @api private
|
430
485
|
def after_perform_retry(*args)
|
486
|
+
return if Resque.inline?
|
431
487
|
log_message 'after_perform_retry, clearing retry key', args
|
432
488
|
clean_retry_key(*args)
|
433
489
|
end
|
@@ -443,16 +499,25 @@ module Resque
|
|
443
499
|
#
|
444
500
|
# @api private
|
445
501
|
def on_failure_retry(exception, *args)
|
502
|
+
return if Resque.inline?
|
446
503
|
log_message 'on_failure_retry', args, exception
|
447
504
|
if exception.is_a?(Resque::DirtyExit)
|
448
505
|
# This hook is called from a worker processes, not the job process
|
449
506
|
# that failed with a DirtyExit, so @retry_attempt wasn't set yet
|
450
507
|
@retry_attempt = Resque.redis.get(redis_retry_key(*args)).to_i
|
451
|
-
elsif
|
508
|
+
elsif instance_variable_defined?(:@on_failure_retry_hook_already_called) && \
|
509
|
+
@on_failure_retry_hook_already_called
|
452
510
|
log_message 'on_failure_retry_hook_already_called', args, exception
|
453
511
|
return
|
454
512
|
end
|
455
513
|
|
514
|
+
# If we are "ignoring" the exception, then we decrement the retry
|
515
|
+
# counter, so that the current attempt didn't count toward the retry
|
516
|
+
# counter.
|
517
|
+
if ignore_exceptions.include?(exception.class)
|
518
|
+
@retry_attempt = Resque.redis.decr(redis_retry_key(*args))
|
519
|
+
end
|
520
|
+
|
456
521
|
if retry_criteria_valid?(exception, *args)
|
457
522
|
try_again(exception, *args)
|
458
523
|
else
|
@@ -584,6 +649,10 @@ module Resque
|
|
584
649
|
end
|
585
650
|
end
|
586
651
|
|
652
|
+
def ignore_exceptions
|
653
|
+
@ignore_exceptions ||= []
|
654
|
+
end
|
655
|
+
|
587
656
|
# Helper to call functions that may be passed as Symbols or Procs. If
|
588
657
|
# a symbol, it is assumed to refer to a method that is already defined
|
589
658
|
# on this class.
|
data/lib/resque-retry/server.rb
CHANGED
@@ -66,15 +66,23 @@ module ResqueRetry
|
|
66
66
|
# cancels job retry
|
67
67
|
def cancel_retry(job)
|
68
68
|
klass = get_class(job)
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
69
|
+
if klass
|
70
|
+
retry_key = retry_key_for_job(job)
|
71
|
+
Resque.remove_delayed(klass, *job['args'])
|
72
|
+
Resque.redis.del("failure-#{retry_key}")
|
73
|
+
Resque.redis.del(retry_key)
|
74
|
+
else
|
75
|
+
raise 'cannot cancel, job not found'
|
76
|
+
end
|
73
77
|
end
|
74
78
|
|
75
79
|
private
|
76
80
|
def get_class(job)
|
77
|
-
|
81
|
+
begin
|
82
|
+
Resque::Job.new(nil, nil).constantize(job['class'])
|
83
|
+
rescue
|
84
|
+
nil
|
85
|
+
end
|
78
86
|
end
|
79
87
|
end
|
80
88
|
|
data/lib/resque-retry/version.rb
CHANGED
data/resque-retry.gemspec
CHANGED
@@ -25,18 +25,17 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.homepage = 'http://github.com/lantins/resque-retry'
|
26
26
|
s.license = 'MIT'
|
27
27
|
|
28
|
-
s.has_rdoc = false
|
29
28
|
s.files = `git ls-files`.split($/)
|
30
29
|
s.require_paths = %w[lib]
|
31
30
|
|
32
|
-
s.add_dependency('resque', '
|
33
|
-
s.add_dependency('resque-scheduler', '
|
31
|
+
s.add_dependency('resque', '>= 1.25', '< 3.0')
|
32
|
+
s.add_dependency('resque-scheduler', '>= 4.0', '<6.0')
|
34
33
|
|
35
34
|
s.add_development_dependency('rake', '~> 10.3')
|
36
35
|
s.add_development_dependency('minitest', '~> 5.5')
|
37
36
|
s.add_development_dependency('rack-test', '~> 0.6')
|
38
37
|
s.add_development_dependency('yard', '~> 0.8')
|
39
|
-
s.add_development_dependency('json', '~>
|
38
|
+
s.add_development_dependency('json', '~> 2.0')
|
40
39
|
s.add_development_dependency('simplecov', '~> 0.9')
|
41
40
|
s.add_development_dependency('mocha', '~> 1.1')
|
42
41
|
end
|
@@ -128,4 +128,11 @@ class ExponentialBackoffTest < Minitest::Test
|
|
128
128
|
assert_equal 4, Resque.info[:failed], 'failed jobs'
|
129
129
|
assert_equal 0, Resque.info[:pending], 'pending jobs'
|
130
130
|
end
|
131
|
+
|
132
|
+
def test_backoff_with_expiration
|
133
|
+
Resque.redis.expects(:expire)
|
134
|
+
|
135
|
+
Resque.enqueue(ExponentialBackoffWithExpiryJob)
|
136
|
+
perform_next_job(@worker)
|
137
|
+
end
|
131
138
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class IgnoreExceptionsTest < Minitest::Test
|
4
|
+
def setup
|
5
|
+
Resque.redis.flushall
|
6
|
+
@worker = Resque::Worker.new(:testing)
|
7
|
+
@worker.register_worker
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_ignore_exceptions
|
11
|
+
Resque.enqueue(IgnoreExceptionsJob)
|
12
|
+
retry_key = IgnoreExceptionsJob.redis_retry_key
|
13
|
+
|
14
|
+
IgnoreExceptionsJob.stubs(:perform).raises(AnotherCustomException)
|
15
|
+
perform_next_job(@worker)
|
16
|
+
assert_equal '0', Resque.redis.get(retry_key), 'retry counter'
|
17
|
+
|
18
|
+
IgnoreExceptionsJob.stubs(:perform).raises(AnotherCustomException)
|
19
|
+
perform_next_job(@worker)
|
20
|
+
assert_equal '1', Resque.redis.get(retry_key), 'retry counter'
|
21
|
+
|
22
|
+
IgnoreExceptionsJob.stubs(:perform).raises(CustomException)
|
23
|
+
perform_next_job(@worker)
|
24
|
+
assert_equal '1', Resque.redis.get(retry_key), 'retry counter'
|
25
|
+
end
|
26
|
+
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
require_relative './test_helper'
|
2
2
|
|
3
3
|
# Mock failure backend for testing MultipleWithRetrySuppression
|
4
4
|
class MockFailureBackend < Resque::Failure::Base
|
@@ -14,7 +14,6 @@ class MockFailureBackend < Resque::Failure::Base
|
|
14
14
|
end
|
15
15
|
|
16
16
|
class MultipleFailureTest < Minitest::Test
|
17
|
-
|
18
17
|
def setup
|
19
18
|
Resque.redis.flushall
|
20
19
|
@worker = Resque::Worker.new(:testing)
|
@@ -26,9 +25,8 @@ class MultipleFailureTest < Minitest::Test
|
|
26
25
|
Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
|
27
26
|
end
|
28
27
|
|
29
|
-
def failure_key_for(klass)
|
30
|
-
args
|
31
|
-
key = 'failure-' + klass.redis_retry_key(args)
|
28
|
+
def failure_key_for(klass, *args)
|
29
|
+
Resque::Failure::MultipleWithRetrySuppression.failure_key(klass.redis_retry_key(args))
|
32
30
|
end
|
33
31
|
|
34
32
|
def test_failure_is_passed_on_when_job_class_not_found
|
@@ -41,7 +39,9 @@ class MultipleFailureTest < Minitest::Test
|
|
41
39
|
perform_next_job(@worker)
|
42
40
|
|
43
41
|
assert_equal 1, MockFailureBackend.errors.count, 'should have one error'
|
44
|
-
|
42
|
+
|
43
|
+
uninitialized_constant_pattern = /uninitialized constant.* LimitThreeJobTemp/
|
44
|
+
assert_match uninitialized_constant_pattern, MockFailureBackend.errors.first
|
45
45
|
end
|
46
46
|
|
47
47
|
def test_last_failure_is_saved_in_redis_if_delay
|
@@ -53,6 +53,22 @@ class MultipleFailureTest < Minitest::Test
|
|
53
53
|
assert Resque.redis.exists(key)
|
54
54
|
end
|
55
55
|
|
56
|
+
def test_retry_delay_is_calculated_with_custom_calculation
|
57
|
+
delay = 5
|
58
|
+
Resque.enqueue(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
|
59
|
+
perform_next_job(@worker)
|
60
|
+
|
61
|
+
key = failure_key_for(DynamicDelayedJobOnExceptionAndArgs, delay.to_s)
|
62
|
+
ttl = Resque.redis.ttl(key)
|
63
|
+
assert Resque.redis.exists(key)
|
64
|
+
assert MockFailureBackend.errors.size == 0
|
65
|
+
|
66
|
+
# expiration on failure_key is set to 2x the delay
|
67
|
+
# to ensure the customized delay is properly calculated using
|
68
|
+
# dynamic retry_delay method on the job
|
69
|
+
assert ttl > delay && delay <= (delay * 2)
|
70
|
+
end
|
71
|
+
|
56
72
|
def test_retry_key_splatting_args
|
57
73
|
# were expecting this to be called three times:
|
58
74
|
# - once when we queue the job to try again
|
@@ -75,7 +91,7 @@ class MultipleFailureTest < Minitest::Test
|
|
75
91
|
|
76
92
|
Resque.enqueue(LimitThreeJobDelay1Hour)
|
77
93
|
perform_next_job(@worker)
|
78
|
-
assert
|
94
|
+
assert [false, 0].include?(Resque.redis.exists(key) || false), 'key should have been removed.'
|
79
95
|
end
|
80
96
|
|
81
97
|
def test_last_failure_has_double_delay_redis_expiry_if_delay
|
@@ -93,7 +109,7 @@ class MultipleFailureTest < Minitest::Test
|
|
93
109
|
|
94
110
|
# I don't like this, but...
|
95
111
|
key = failure_key_for(LimitThreeJob)
|
96
|
-
assert
|
112
|
+
assert [false, 0].include?(Resque.redis.exists(key) || false), 'key should have not been added.'
|
97
113
|
end
|
98
114
|
|
99
115
|
def test_errors_are_suppressed_up_to_retry_limit
|
@@ -141,6 +157,23 @@ class MultipleFailureTest < Minitest::Test
|
|
141
157
|
end
|
142
158
|
end
|
143
159
|
|
160
|
+
def test_redis_exists_returns_integer
|
161
|
+
return if !Redis.respond_to?(:exists_returns_integer)
|
162
|
+
|
163
|
+
Resque.enqueue(RetryDefaultsJob)
|
164
|
+
|
165
|
+
original = Redis.exists_returns_integer
|
166
|
+
Redis.exists_returns_integer = true
|
167
|
+
|
168
|
+
3.times do
|
169
|
+
perform_next_job(@worker)
|
170
|
+
end
|
171
|
+
|
172
|
+
Redis.exists_returns_integer = original
|
173
|
+
|
174
|
+
assert_equal 1, MockFailureBackend.errors.size
|
175
|
+
end
|
176
|
+
|
144
177
|
def teardown
|
145
178
|
Resque::Failure.backend = @old_failure_backend
|
146
179
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class ResqueInlineTest < Minitest::Test
|
4
|
+
def setup
|
5
|
+
Resque.inline = true
|
6
|
+
Resque.expects(:redis).never
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
Resque.inline = false
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_runs_inline
|
14
|
+
GoodJob.expects :perform
|
15
|
+
Resque.enqueue(GoodJob)
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_fails_inline
|
19
|
+
assert_raises CustomException do
|
20
|
+
Resque.enqueue(RetryCustomExceptionsJob, 'CustomException')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|