onyx-resque-retry 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY.md +33 -0
- data/LICENSE +21 -0
- data/README.md +289 -0
- data/Rakefile +25 -0
- data/lib/resque-retry.rb +6 -0
- data/lib/resque-retry/server.rb +51 -0
- data/lib/resque-retry/server/views/retry.erb +48 -0
- data/lib/resque-retry/server/views/retry_timestamp.erb +59 -0
- data/lib/resque/failure/multiple_with_retry_suppression.rb +93 -0
- data/lib/resque/plugins/exponential_backoff.rb +64 -0
- data/lib/resque/plugins/retry.rb +221 -0
- data/test/exponential_backoff_test.rb +62 -0
- data/test/multiple_failure_test.rb +86 -0
- data/test/redis-test.conf +132 -0
- data/test/resque_test.rb +18 -0
- data/test/retry_criteria_test.rb +75 -0
- data/test/retry_inheriting_checks_test.rb +33 -0
- data/test/retry_test.rb +173 -0
- data/test/test_helper.rb +78 -0
- data/test/test_jobs.rb +280 -0
- metadata +189 -0
@@ -0,0 +1,59 @@
|
|
1
|
+
<% timestamp = params[:timestamp].to_i %>
|
2
|
+
|
3
|
+
<h1>
|
4
|
+
Delayed Jobs scheduled for <%= format_time(Time.at(timestamp)) %>
|
5
|
+
(with Retry Information)
|
6
|
+
</h1>
|
7
|
+
|
8
|
+
<p class="intro">
|
9
|
+
This list below contains the delayed jobs scheduled for the current
|
10
|
+
timestamp, with retry information.
|
11
|
+
</p>
|
12
|
+
|
13
|
+
<p class="sub">
|
14
|
+
Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of
|
15
|
+
<b><%= size = resque.delayed_timestamp_size(timestamp) %></b> jobs
|
16
|
+
</p>
|
17
|
+
|
18
|
+
<table class="jobs">
|
19
|
+
<tr>
|
20
|
+
<th>Class</th>
|
21
|
+
<th>Args</th>
|
22
|
+
<th>Retry Attempts</th>
|
23
|
+
<th>Exception</th>
|
24
|
+
<th>Backtrace</th>
|
25
|
+
</tr>
|
26
|
+
<% jobs = resque.delayed_timestamp_peek(timestamp, start, 20) %>
|
27
|
+
<% jobs.each do |job| %>
|
28
|
+
<% retry_key = retry_key_for_job(job) %>
|
29
|
+
<% retry_attempts = retry_attempts_for_job(job) %>
|
30
|
+
<tr>
|
31
|
+
<td class="class"><%= h job['class'] %></td>
|
32
|
+
<td class="args"><%= h job['args'].inspect %></td>
|
33
|
+
<% if retry_attempts.nil? %>
|
34
|
+
<td colspan="3"><i>n/a - normal delayed job</i></td>
|
35
|
+
<% else %>
|
36
|
+
<td><%= retry_attempts %></td>
|
37
|
+
<% failure = retry_failure_details(retry_key) %>
|
38
|
+
<td><code><%= failure['exception'] %></code></td>
|
39
|
+
<td class="error">
|
40
|
+
<% if failure['backtrace'] %>
|
41
|
+
<a href="#" class="backtrace"><%= h(failure['error']) %></a>
|
42
|
+
<pre style="display:none"><%= h failure['backtrace'].join("\n") %></pre>
|
43
|
+
<% else %>
|
44
|
+
<%= h failure['error'] %>
|
45
|
+
<% end %>
|
46
|
+
</td>
|
47
|
+
<% end %>
|
48
|
+
</tr>
|
49
|
+
<% end %>
|
50
|
+
<% if jobs.empty? %>
|
51
|
+
<tr>
|
52
|
+
<td class="no-data" colspan="5">
|
53
|
+
There are no pending jobs scheduled for this time.
|
54
|
+
</td>
|
55
|
+
</tr>
|
56
|
+
<% end %>
|
57
|
+
</table>
|
58
|
+
|
59
|
+
<%= partial :next_more, :start => start, :size => size %>
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'resque/failure/multiple'
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Failure
|
5
|
+
|
6
|
+
# A multiple failure backend, with retry suppression.
|
7
|
+
#
|
8
|
+
# For example: if you had a job that could retry 5 times, your failure
|
9
|
+
# backends are not notified unless the _final_ retry attempt also fails.
|
10
|
+
#
|
11
|
+
# Example:
|
12
|
+
#
|
13
|
+
# require 'resque-retry'
|
14
|
+
# require 'resque/failure/redis'
|
15
|
+
#
|
16
|
+
# Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
|
17
|
+
# Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
|
18
|
+
#
|
19
|
+
class MultipleWithRetrySuppression < Multiple
|
20
|
+
include Resque::Helpers
|
21
|
+
|
22
|
+
module CleanupHooks
|
23
|
+
# Resque after_perform hook.
|
24
|
+
#
|
25
|
+
# Deletes retry failure information from Redis.
|
26
|
+
def after_perform_retry_failure_cleanup(*args)
|
27
|
+
retry_key = redis_retry_key(*args)
|
28
|
+
failure_key = Resque::Failure::MultipleWithRetrySuppression.failure_key(retry_key)
|
29
|
+
Resque.redis.del(failure_key)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Called when the job fails.
|
34
|
+
#
|
35
|
+
# If the job will retry, suppress the failure from the other backends.
|
36
|
+
# Store the lastest failure information in redis, used by the web
|
37
|
+
# interface.
|
38
|
+
def save
|
39
|
+
unless retryable? && retrying?
|
40
|
+
cleanup_retry_failure_log!
|
41
|
+
super
|
42
|
+
else
|
43
|
+
data = {
|
44
|
+
:failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
|
45
|
+
:payload => payload,
|
46
|
+
:exception => exception.class.to_s,
|
47
|
+
:error => exception.to_s,
|
48
|
+
:backtrace => Array(exception.backtrace),
|
49
|
+
:worker => worker.to_s,
|
50
|
+
:queue => queue
|
51
|
+
}
|
52
|
+
|
53
|
+
# Register cleanup hooks.
|
54
|
+
unless klass.respond_to?(:after_perform_retry_failure_cleanup)
|
55
|
+
klass.send(:extend, CleanupHooks)
|
56
|
+
end
|
57
|
+
|
58
|
+
redis[failure_key] = Resque.encode(data)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Expose this for the hook's use.
|
63
|
+
def self.failure_key(retry_key)
|
64
|
+
'failure_' + retry_key
|
65
|
+
end
|
66
|
+
|
67
|
+
protected
|
68
|
+
def klass
|
69
|
+
constantize(payload['class'])
|
70
|
+
end
|
71
|
+
|
72
|
+
def retry_key
|
73
|
+
klass.redis_retry_key(payload['args'])
|
74
|
+
end
|
75
|
+
|
76
|
+
def failure_key
|
77
|
+
self.class.failure_key(retry_key)
|
78
|
+
end
|
79
|
+
|
80
|
+
def retryable?
|
81
|
+
klass.respond_to?(:redis_retry_key)
|
82
|
+
end
|
83
|
+
|
84
|
+
def retrying?
|
85
|
+
redis.exists(retry_key)
|
86
|
+
end
|
87
|
+
|
88
|
+
def cleanup_retry_failure_log!
|
89
|
+
redis.del(failure_key) if retryable?
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
|
4
|
+
# If you want your job to retry on failure using a varying delay, simply
|
5
|
+
# extend your module/class with this module:
|
6
|
+
#
|
7
|
+
# class DeliverSMS
|
8
|
+
# extend Resque::Plugins::ExponentialBackoff
|
9
|
+
# @queue = :mt_messages
|
10
|
+
#
|
11
|
+
# def self.perform(mt_id, mobile_number, message)
|
12
|
+
# heavy_lifting
|
13
|
+
# end
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# Easily do something custom:
|
17
|
+
#
|
18
|
+
# class DeliverSMS
|
19
|
+
# extend Resque::Plugins::ExponentialBackoff
|
20
|
+
# @queue = :mt_messages
|
21
|
+
#
|
22
|
+
# @retry_limit = 4
|
23
|
+
#
|
24
|
+
# # retry delay in seconds; [0] => 1st retry, [1] => 2nd..4th retry.
|
25
|
+
# @backoff_strategy = [0, 60]
|
26
|
+
#
|
27
|
+
# # used to build redis key, for counting job attempts.
|
28
|
+
# def self.identifier(mt_id, mobile_number, message)
|
29
|
+
# "#{mobile_number}:#{mt_id}"
|
30
|
+
# end
|
31
|
+
#
|
32
|
+
# self.perform(mt_id, mobile_number, message)
|
33
|
+
# heavy_lifting
|
34
|
+
# end
|
35
|
+
# end
|
36
|
+
#
|
37
|
+
module ExponentialBackoff
|
38
|
+
include Resque::Plugins::Retry
|
39
|
+
|
40
|
+
# Defaults to the number of delays in the backoff strategy.
|
41
|
+
#
|
42
|
+
# @return [Number] maximum number of retries
|
43
|
+
def retry_limit
|
44
|
+
@retry_limit ||= backoff_strategy.length
|
45
|
+
end
|
46
|
+
|
47
|
+
# Selects the delay from the backoff strategy.
|
48
|
+
#
|
49
|
+
# @return [Number] seconds to delay until the next retry.
|
50
|
+
def retry_delay
|
51
|
+
backoff_strategy[retry_attempt] || backoff_strategy.last
|
52
|
+
end
|
53
|
+
|
54
|
+
# @abstract
|
55
|
+
# The backoff strategy is used to vary the delay between retry attempts.
|
56
|
+
#
|
57
|
+
# @return [Array] array of delays. index = retry attempt
|
58
|
+
def backoff_strategy
|
59
|
+
@backoff_strategy ||= [0, 60, 600, 3600, 10_800, 21_600]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,221 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
|
4
|
+
# If you want your job to retry on failure, simply extend your module/class
|
5
|
+
# with this module:
|
6
|
+
#
|
7
|
+
# class DeliverWebHook
|
8
|
+
# extend Resque::Plugins::Retry # allows 1 retry by default.
|
9
|
+
# @queue = :web_hooks
|
10
|
+
#
|
11
|
+
# def self.perform(url, hook_id, hmac_key)
|
12
|
+
# heavy_lifting
|
13
|
+
# end
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# Easily do something custom:
|
17
|
+
#
|
18
|
+
# class DeliverWebHook
|
19
|
+
# extend Resque::Plugins::Retry
|
20
|
+
# @queue = :web_hooks
|
21
|
+
#
|
22
|
+
# @retry_limit = 8 # default: 1
|
23
|
+
# @retry_delay = 60 # default: 0
|
24
|
+
#
|
25
|
+
# # used to build redis key, for counting job attempts.
|
26
|
+
# def self.identifier(url, hook_id, hmac_key)
|
27
|
+
# "#{url}-#{hook_id}"
|
28
|
+
# end
|
29
|
+
#
|
30
|
+
# def self.perform(url, hook_id, hmac_key)
|
31
|
+
# heavy_lifting
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
#
|
35
|
+
module Retry
|
36
|
+
|
37
|
+
# Copy retry criteria checks on inheritance.
|
38
|
+
def inherited(subclass)
|
39
|
+
super(subclass)
|
40
|
+
subclass.instance_variable_set("@retry_criteria_checks", retry_criteria_checks.dup)
|
41
|
+
end
|
42
|
+
|
43
|
+
# @abstract You may override to implement a custom identifier,
|
44
|
+
# you should consider doing this if your job arguments
|
45
|
+
# are many/long or may not cleanly cleanly to strings.
|
46
|
+
#
|
47
|
+
# Builds an identifier using the job arguments. This identifier
|
48
|
+
# is used as part of the redis key.
|
49
|
+
#
|
50
|
+
# @param [Array] args job arguments
|
51
|
+
# @return [String] job identifier
|
52
|
+
def identifier(*args)
|
53
|
+
args_string = args.join('-')
|
54
|
+
args_string.empty? ? nil : args_string
|
55
|
+
end
|
56
|
+
|
57
|
+
# Builds the redis key to be used for keeping state of the job
|
58
|
+
# attempts.
|
59
|
+
#
|
60
|
+
# @return [String] redis key
|
61
|
+
def redis_retry_key(*args)
|
62
|
+
['resque-retry', name, identifier(*args)].compact.join(":").gsub(/\s/, '')
|
63
|
+
end
|
64
|
+
|
65
|
+
# Maximum number of retrys we can attempt to successfully perform the job.
|
66
|
+
# A retry limit of 0 or below will retry forever.
|
67
|
+
#
|
68
|
+
# @return [Fixnum]
|
69
|
+
def retry_limit
|
70
|
+
@retry_limit ||= 1
|
71
|
+
end
|
72
|
+
|
73
|
+
# Number of retry attempts used to try and perform the job.
|
74
|
+
#
|
75
|
+
# The real value is kept in Redis, it is accessed and incremented using
|
76
|
+
# a before_perform hook.
|
77
|
+
#
|
78
|
+
# @return [Fixnum] number of attempts
|
79
|
+
def retry_attempt
|
80
|
+
@retry_attempt ||= 0
|
81
|
+
end
|
82
|
+
|
83
|
+
# @abstract
|
84
|
+
# Number of seconds to delay until the job is retried.
|
85
|
+
#
|
86
|
+
# @return [Number] number of seconds to delay
|
87
|
+
def retry_delay
|
88
|
+
@retry_delay ||= 0
|
89
|
+
end
|
90
|
+
|
91
|
+
# @abstract
|
92
|
+
# Modify the arguments used to retry the job. Use this to do something
|
93
|
+
# other than try the exact same job again.
|
94
|
+
#
|
95
|
+
# @return [Array] new job arguments
|
96
|
+
def args_for_retry(*args)
|
97
|
+
args
|
98
|
+
end
|
99
|
+
|
100
|
+
# Convenience method to test whether you may retry on a given exception.
|
101
|
+
#
|
102
|
+
# @return [Boolean]
|
103
|
+
def retry_exception?(exception)
|
104
|
+
return true if retry_exceptions.nil?
|
105
|
+
!! retry_exceptions.any? { |ex| ex >= exception }
|
106
|
+
end
|
107
|
+
|
108
|
+
# @abstract
|
109
|
+
# Controls what exceptions may be retried.
|
110
|
+
#
|
111
|
+
# Default: `nil` - this will retry all exceptions.
|
112
|
+
#
|
113
|
+
# @return [Array, nil]
|
114
|
+
def retry_exceptions
|
115
|
+
@retry_exceptions ||= nil
|
116
|
+
end
|
117
|
+
|
118
|
+
# Test if the retry criteria is valid.
|
119
|
+
#
|
120
|
+
# @param [Exception] exception
|
121
|
+
# @param [Array] args job arguments
|
122
|
+
# @return [Boolean]
|
123
|
+
def retry_criteria_valid?(exception, *args)
|
124
|
+
# if the retry limit was reached, dont bother checking anything else.
|
125
|
+
return false if retry_limit_reached?
|
126
|
+
|
127
|
+
# We always want to retry if the exception matches.
|
128
|
+
should_retry = retry_exception?(exception.class)
|
129
|
+
|
130
|
+
# call user retry criteria check blocks.
|
131
|
+
retry_criteria_checks.each do |criteria_check|
|
132
|
+
should_retry ||= !!criteria_check.call(exception, *args)
|
133
|
+
end
|
134
|
+
|
135
|
+
should_retry
|
136
|
+
end
|
137
|
+
|
138
|
+
# Retry criteria checks.
|
139
|
+
#
|
140
|
+
# @return [Array]
|
141
|
+
def retry_criteria_checks
|
142
|
+
@retry_criteria_checks ||= []
|
143
|
+
@retry_criteria_checks
|
144
|
+
end
|
145
|
+
|
146
|
+
# Test if the retry limit has been reached.
|
147
|
+
#
|
148
|
+
# @return [Boolean]
|
149
|
+
def retry_limit_reached?
|
150
|
+
if retry_limit > 0
|
151
|
+
return true if retry_attempt >= retry_limit
|
152
|
+
end
|
153
|
+
false
|
154
|
+
end
|
155
|
+
|
156
|
+
# Register a retry criteria check callback to be run before retrying
|
157
|
+
# the job again.
|
158
|
+
#
|
159
|
+
# If any callback returns `true`, the job will be retried.
|
160
|
+
#
|
161
|
+
# @example Using a custom retry criteria check.
|
162
|
+
#
|
163
|
+
# retry_criteria_check do |exception, *args|
|
164
|
+
# if exception.message =~ /InvalidJobId/
|
165
|
+
# # don't retry if we got passed a invalid job id.
|
166
|
+
# false
|
167
|
+
# else
|
168
|
+
# true
|
169
|
+
# end
|
170
|
+
# end
|
171
|
+
#
|
172
|
+
# @yield [exception, *args]
|
173
|
+
# @yieldparam exception [Exception] the exception that was raised
|
174
|
+
# @yieldparam args [Array] job arguments
|
175
|
+
# @yieldreturn [Boolean] false == dont retry, true = can retry
|
176
|
+
def retry_criteria_check(&block)
|
177
|
+
retry_criteria_checks << block
|
178
|
+
end
|
179
|
+
|
180
|
+
# Will retry the job.
|
181
|
+
def try_again(*args)
|
182
|
+
@retry_job_class ||= self
|
183
|
+
if retry_delay <= 0
|
184
|
+
# If the delay is 0, no point passing it through the scheduler
|
185
|
+
Resque.enqueue(@retry_job_class, *args_for_retry(*args))
|
186
|
+
else
|
187
|
+
Resque.enqueue_in(retry_delay, @retry_job_class, *args_for_retry(*args))
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
# Resque before_perform hook.
|
192
|
+
#
|
193
|
+
# Increments and sets the `@retry_attempt` count.
|
194
|
+
def before_perform_retry(*args)
|
195
|
+
retry_key = redis_retry_key(*args)
|
196
|
+
Resque.redis.setnx(retry_key, -1) # default to -1 if not set.
|
197
|
+
@retry_attempt = Resque.redis.incr(retry_key) # increment by 1.
|
198
|
+
end
|
199
|
+
|
200
|
+
# Resque after_perform hook.
|
201
|
+
#
|
202
|
+
# Deletes retry attempt count from Redis.
|
203
|
+
def after_perform_retry(*args)
|
204
|
+
Resque.redis.del(redis_retry_key(*args))
|
205
|
+
end
|
206
|
+
|
207
|
+
# Resque on_failure hook.
|
208
|
+
#
|
209
|
+
# Checks if our retry criteria is valid, if it is we try again.
|
210
|
+
# Otherwise the retry attempt count is deleted from Redis.
|
211
|
+
def on_failure_retry(exception, *args)
|
212
|
+
if retry_criteria_valid?(exception, *args)
|
213
|
+
try_again(*args)
|
214
|
+
else
|
215
|
+
Resque.redis.del(redis_retry_key(*args))
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class ExponentialBackoffTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Resque.redis.flushall
|
6
|
+
@worker = Resque::Worker.new(:testing)
|
7
|
+
@worker.register_worker
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_resque_plugin_lint
|
11
|
+
assert_nothing_raised do
|
12
|
+
Resque::Plugin.lint(Resque::Plugins::ExponentialBackoff)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_default_backoff_strategy
|
17
|
+
now = Time.now
|
18
|
+
Resque.enqueue(ExponentialBackoffJob)
|
19
|
+
|
20
|
+
perform_next_job @worker
|
21
|
+
assert_equal 1, Resque.info[:processed], '1 processed job'
|
22
|
+
assert_equal 1, Resque.info[:failed], 'first ever run, and it should of failed, but never retried'
|
23
|
+
assert_equal 1, Resque.info[:pending], '1 pending job, because it never hits the scheduler'
|
24
|
+
|
25
|
+
perform_next_job @worker
|
26
|
+
assert_equal 2, Resque.info[:processed], '2nd run, but first retry'
|
27
|
+
assert_equal 2, Resque.info[:failed], 'should of failed again, this is the first retry attempt'
|
28
|
+
assert_equal 0, Resque.info[:pending], '0 pending jobs, it should be in the delayed queue'
|
29
|
+
|
30
|
+
delayed = Resque.delayed_queue_peek(0, 1)
|
31
|
+
assert_equal now.to_i + 60, delayed[0], '2nd delay' # the first had a zero delay.
|
32
|
+
|
33
|
+
5.times do
|
34
|
+
Resque.enqueue(ExponentialBackoffJob)
|
35
|
+
perform_next_job @worker
|
36
|
+
end
|
37
|
+
|
38
|
+
delayed = Resque.delayed_queue_peek(0, 5)
|
39
|
+
assert_equal now.to_i + 600, delayed[1], '3rd delay'
|
40
|
+
assert_equal now.to_i + 3600, delayed[2], '4th delay'
|
41
|
+
assert_equal now.to_i + 10_800, delayed[3], '5th delay'
|
42
|
+
assert_equal now.to_i + 21_600, delayed[4], '6th delay'
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_custom_backoff_strategy
|
46
|
+
now = Time.now
|
47
|
+
4.times do
|
48
|
+
Resque.enqueue(CustomExponentialBackoffJob, 'http://lividpenguin.com', 1305, 'cd8079192d379dc612f17c660591a6cfb05f1dda')
|
49
|
+
perform_next_job @worker
|
50
|
+
end
|
51
|
+
|
52
|
+
delayed = Resque.delayed_queue_peek(0, 3)
|
53
|
+
assert_equal now.to_i + 10, delayed[0], '1st delay'
|
54
|
+
assert_equal now.to_i + 20, delayed[1], '2nd delay'
|
55
|
+
assert_equal now.to_i + 30, delayed[2], '3rd delay'
|
56
|
+
assert_equal 2, Resque.delayed_timestamp_size(delayed[2]), '4th delay should share delay with 3rd'
|
57
|
+
|
58
|
+
assert_equal 4, Resque.info[:processed], 'processed jobs'
|
59
|
+
assert_equal 4, Resque.info[:failed], 'failed jobs'
|
60
|
+
assert_equal 0, Resque.info[:pending], 'pending jobs'
|
61
|
+
end
|
62
|
+
end
|