sidekiq 6.0.7 → 6.4.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +189 -2
- data/LICENSE +3 -3
- data/README.md +11 -10
- data/bin/sidekiq +8 -3
- data/bin/sidekiqload +57 -65
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +164 -116
- data/lib/sidekiq/cli.rb +49 -15
- data/lib/sidekiq/client.rb +51 -70
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +32 -23
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +32 -33
- data/lib/sidekiq/job_util.rb +67 -0
- data/lib/sidekiq/launcher.rb +113 -54
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +16 -18
- data/lib/sidekiq/middleware/chain.rb +10 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +4 -4
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +31 -31
- data/lib/sidekiq/rails.rb +36 -20
- data/lib/sidekiq/redis_connection.rb +16 -15
- data/lib/sidekiq/scheduled.rb +51 -16
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/util.rb +41 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +39 -33
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +135 -16
- data/lib/sidekiq.rb +33 -17
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +113 -65
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +42 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +51 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +24 -49
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
data/lib/sidekiq/client.rb
CHANGED
@@ -2,9 +2,12 @@
|
|
2
2
|
|
3
3
|
require "securerandom"
|
4
4
|
require "sidekiq/middleware/chain"
|
5
|
+
require "sidekiq/job_util"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
class Client
|
9
|
+
include Sidekiq::JobUtil
|
10
|
+
|
8
11
|
##
|
9
12
|
# Define client-side middleware:
|
10
13
|
#
|
@@ -12,14 +15,14 @@ module Sidekiq
|
|
12
15
|
# client.middleware do |chain|
|
13
16
|
# chain.use MyClientMiddleware
|
14
17
|
# end
|
15
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
16
19
|
#
|
17
20
|
# All client instances default to the globally-defined
|
18
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
19
22
|
#
|
20
23
|
def middleware(&block)
|
21
24
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
25
|
+
if block
|
23
26
|
@chain = @chain.dup
|
24
27
|
yield @chain
|
25
28
|
end
|
@@ -46,16 +49,16 @@ module Sidekiq
|
|
46
49
|
# The main method used to push a job to Redis. Accepts a number of options:
|
47
50
|
#
|
48
51
|
# queue - the named queue to use, default 'default'
|
49
|
-
# class - the
|
52
|
+
# class - the job class to call, required
|
50
53
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
54
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
55
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
56
|
# backtrace - whether to save any error backtrace, default false
|
54
57
|
#
|
55
58
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
-
#
|
59
|
+
# job options. Otherwise, they will be based on the job class's options.
|
57
60
|
#
|
58
|
-
# Any options valid for a
|
61
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
59
62
|
#
|
60
63
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
61
64
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -64,13 +67,15 @@ module Sidekiq
|
|
64
67
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
65
68
|
#
|
66
69
|
# Example:
|
67
|
-
# push('queue' => 'my_queue', 'class' =>
|
70
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
68
71
|
#
|
69
72
|
def push(item)
|
70
73
|
normed = normalize_item(item)
|
71
|
-
payload =
|
72
|
-
|
74
|
+
payload = middleware.invoke(normed["class"], normed, normed["queue"], @redis_pool) do
|
75
|
+
normed
|
76
|
+
end
|
73
77
|
if payload
|
78
|
+
verify_json(payload)
|
74
79
|
raw_push([payload])
|
75
80
|
payload["jid"]
|
76
81
|
end
|
@@ -90,19 +95,25 @@ module Sidekiq
|
|
90
95
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
96
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
97
|
def push_bulk(items)
|
93
|
-
|
94
|
-
|
95
|
-
|
98
|
+
args = items["args"]
|
99
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
100
|
+
return [] if args.empty? # no jobs to push
|
96
101
|
|
97
102
|
at = items.delete("at")
|
98
|
-
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
|
103
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
104
|
+
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
105
|
+
|
106
|
+
jid = items.delete("jid")
|
107
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
99
108
|
|
100
109
|
normed = normalize_item(items)
|
101
|
-
payloads =
|
102
|
-
copy = normed.merge("args" =>
|
110
|
+
payloads = args.map.with_index { |job_args, index|
|
111
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
103
112
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
104
|
-
|
105
|
-
|
113
|
+
result = middleware.invoke(copy["class"], copy, copy["queue"], @redis_pool) do
|
114
|
+
verify_json(copy)
|
115
|
+
copy
|
116
|
+
end
|
106
117
|
result || nil
|
107
118
|
}.compact
|
108
119
|
|
@@ -115,8 +126,8 @@ module Sidekiq
|
|
115
126
|
#
|
116
127
|
# pool = ConnectionPool.new { Redis.new }
|
117
128
|
# Sidekiq::Client.via(pool) do
|
118
|
-
#
|
119
|
-
#
|
129
|
+
# SomeJob.perform_async(1,2,3)
|
130
|
+
# SomeOtherJob.perform_async(1,2,3)
|
120
131
|
# end
|
121
132
|
#
|
122
133
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -141,10 +152,10 @@ module Sidekiq
|
|
141
152
|
end
|
142
153
|
|
143
154
|
# Resque compatibility helpers. Note all helpers
|
144
|
-
# should go through
|
155
|
+
# should go through Sidekiq::Job#client_push.
|
145
156
|
#
|
146
157
|
# Example usage:
|
147
|
-
# Sidekiq::Client.enqueue(
|
158
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
148
159
|
#
|
149
160
|
# Messages are enqueued to the 'default' queue.
|
150
161
|
#
|
@@ -153,14 +164,14 @@ module Sidekiq
|
|
153
164
|
end
|
154
165
|
|
155
166
|
# Example usage:
|
156
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
167
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
157
168
|
#
|
158
169
|
def enqueue_to(queue, klass, *args)
|
159
170
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
160
171
|
end
|
161
172
|
|
162
173
|
# Example usage:
|
163
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
174
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
164
175
|
#
|
165
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
166
177
|
int = interval.to_f
|
@@ -174,7 +185,7 @@ module Sidekiq
|
|
174
185
|
end
|
175
186
|
|
176
187
|
# Example usage:
|
177
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
188
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
178
189
|
#
|
179
190
|
def enqueue_in(interval, klass, *args)
|
180
191
|
klass.perform_in(interval, *args)
|
@@ -185,8 +196,23 @@ module Sidekiq
|
|
185
196
|
|
186
197
|
def raw_push(payloads)
|
187
198
|
@redis_pool.with do |conn|
|
188
|
-
|
189
|
-
|
199
|
+
retryable = true
|
200
|
+
begin
|
201
|
+
conn.pipelined do |pipeline|
|
202
|
+
atomic_push(pipeline, payloads)
|
203
|
+
end
|
204
|
+
rescue Redis::BaseError => ex
|
205
|
+
# 2550 Failover can cause the server to become a replica, need
|
206
|
+
# to disconnect and reopen the socket to get back to the primary.
|
207
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
|
+
# The retry logic is copied from sidekiq.rb
|
210
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
+
conn.disconnect!
|
212
|
+
retryable = false
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
raise
|
190
216
|
end
|
191
217
|
end
|
192
218
|
true
|
@@ -209,50 +235,5 @@ module Sidekiq
|
|
209
235
|
conn.lpush("queue:#{queue}", to_push)
|
210
236
|
end
|
211
237
|
end
|
212
|
-
|
213
|
-
def process_single(worker_class, item)
|
214
|
-
queue = item["queue"]
|
215
|
-
|
216
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
217
|
-
item
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def normalize_item(item)
|
222
|
-
# 6.0.0 push_bulk bug, #4321
|
223
|
-
# TODO Remove after a while...
|
224
|
-
item.delete("at") if item.key?("at") && item["at"].nil?
|
225
|
-
|
226
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
227
|
-
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
228
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
229
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
230
|
-
raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
|
231
|
-
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
232
|
-
|
233
|
-
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
234
|
-
# this allows ActiveJobs to control sidekiq_options too.
|
235
|
-
defaults = normalized_hash(item["class"])
|
236
|
-
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
237
|
-
item = defaults.merge(item)
|
238
|
-
|
239
|
-
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
240
|
-
|
241
|
-
item["class"] = item["class"].to_s
|
242
|
-
item["queue"] = item["queue"].to_s
|
243
|
-
item["jid"] ||= SecureRandom.hex(12)
|
244
|
-
item["created_at"] ||= Time.now.to_f
|
245
|
-
|
246
|
-
item
|
247
|
-
end
|
248
|
-
|
249
|
-
def normalized_hash(item_class)
|
250
|
-
if item_class.is_a?(Class)
|
251
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
252
|
-
item_class.get_sidekiq_options
|
253
|
-
else
|
254
|
-
Sidekiq.default_worker_options
|
255
|
-
end
|
256
|
-
end
|
257
238
|
end
|
258
239
|
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
10
10
|
#
|
11
|
+
# @example
|
11
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
12
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
13
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
12
13
|
#
|
13
14
|
# Please note, this is not recommended as this will serialize the entire
|
14
15
|
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
12
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.delay.delete_inactive
|
13
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
13
14
|
#
|
14
15
|
class DelayedClass
|
15
16
|
include Sidekiq::Worker
|
@@ -10,7 +10,7 @@ module Sidekiq
|
|
10
10
|
def initialize(performable, target, options = {})
|
11
11
|
@performable = performable
|
12
12
|
@target = target
|
13
|
-
@opts = options
|
13
|
+
@opts = options.transform_keys(&:to_s)
|
14
14
|
end
|
15
15
|
|
16
16
|
def method_missing(name, *args)
|
@@ -24,7 +24,9 @@ module Sidekiq
|
|
24
24
|
if marshalled.size > SIZE_LIMIT
|
25
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
26
26
|
end
|
27
|
-
@performable.client_push({"class" => @performable,
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -25,8 +25,10 @@ module Sidekiq
|
|
25
25
|
}
|
26
26
|
|
27
27
|
def initialize(options)
|
28
|
-
|
29
|
-
@
|
28
|
+
raise ArgumentError, "missing queue list" unless options[:queues]
|
29
|
+
@options = options
|
30
|
+
@strictly_ordered_queues = !!@options[:strict]
|
31
|
+
@queues = @options[:queues].map { |q| "queue:#{q}" }
|
30
32
|
if @strictly_ordered_queues
|
31
33
|
@queues.uniq!
|
32
34
|
@queues << TIMEOUT
|
@@ -34,28 +36,19 @@ module Sidekiq
|
|
34
36
|
end
|
35
37
|
|
36
38
|
def retrieve_work
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
# data from the first queue that has pending elements. We
|
44
|
-
# recreate the queue command each time we invoke Redis#brpop
|
45
|
-
# to honor weights and avoid queue starvation.
|
46
|
-
def queues_cmd
|
47
|
-
if @strictly_ordered_queues
|
48
|
-
@queues
|
49
|
-
else
|
50
|
-
queues = @queues.shuffle!.uniq
|
51
|
-
queues << TIMEOUT
|
52
|
-
queues
|
39
|
+
qs = queues_cmd
|
40
|
+
# 4825 Sidekiq Pro with all queues paused will return an
|
41
|
+
# empty set of queues with a trailing TIMEOUT value.
|
42
|
+
if qs.size <= 1
|
43
|
+
sleep(TIMEOUT)
|
44
|
+
return nil
|
53
45
|
end
|
46
|
+
|
47
|
+
work = Sidekiq.redis { |conn| conn.brpop(*qs) }
|
48
|
+
UnitOfWork.new(*work) if work
|
54
49
|
end
|
55
50
|
|
56
|
-
|
57
|
-
# an instance method will make it async to the Fetcher actor
|
58
|
-
def self.bulk_requeue(inprogress, options)
|
51
|
+
def bulk_requeue(inprogress, options)
|
59
52
|
return if inprogress.empty?
|
60
53
|
|
61
54
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
@@ -66,9 +59,9 @@ module Sidekiq
|
|
66
59
|
end
|
67
60
|
|
68
61
|
Sidekiq.redis do |conn|
|
69
|
-
conn.pipelined do
|
62
|
+
conn.pipelined do |pipeline|
|
70
63
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
|
64
|
+
pipeline.rpush(queue, jobs)
|
72
65
|
end
|
73
66
|
end
|
74
67
|
end
|
@@ -76,5 +69,21 @@ module Sidekiq
|
|
76
69
|
rescue => ex
|
77
70
|
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
78
71
|
end
|
72
|
+
|
73
|
+
# Creating the Redis#brpop command takes into account any
|
74
|
+
# configured queue weights. By default Redis#brpop returns
|
75
|
+
# data from the first queue that has pending elements. We
|
76
|
+
# recreate the queue command each time we invoke Redis#brpop
|
77
|
+
# to honor weights and avoid queue starvation.
|
78
|
+
def queues_cmd
|
79
|
+
if @strictly_ordered_queues
|
80
|
+
@queues
|
81
|
+
else
|
82
|
+
permute = @queues.shuffle
|
83
|
+
permute.uniq!
|
84
|
+
permute << TIMEOUT
|
85
|
+
permute
|
86
|
+
end
|
87
|
+
end
|
79
88
|
end
|
80
89
|
end
|
data/lib/sidekiq/job.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "sidekiq/worker"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
|
5
|
+
# Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
|
6
|
+
#
|
7
|
+
# The term "worker" is too generic and overly confusing, used in several
|
8
|
+
# different contexts meaning different things. Many people call a Sidekiq
|
9
|
+
# process a "worker". Some people call the thread that executes jobs a
|
10
|
+
# "worker". This change brings Sidekiq closer to ActiveJob where your job
|
11
|
+
# classes extend ApplicationJob.
|
12
|
+
Job = Worker
|
13
|
+
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -12,46 +12,34 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
yield
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
end
|
15
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
16
|
+
@logger.info("done")
|
18
17
|
rescue Exception
|
19
|
-
|
20
|
-
|
21
|
-
end
|
18
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
19
|
+
@logger.info("fail")
|
22
20
|
|
23
21
|
raise
|
24
22
|
end
|
25
23
|
|
26
24
|
def prepare(job_hash, &block)
|
27
|
-
level = job_hash["log_level"]
|
28
|
-
if level
|
29
|
-
@logger.log_at(level) do
|
30
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
-
end
|
32
|
-
else
|
33
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def job_hash_context(job_hash)
|
38
25
|
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
39
26
|
# attribute to expose the underlying thing.
|
40
27
|
h = {
|
41
|
-
class: job_hash["wrapped"] || job_hash["class"],
|
28
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
|
42
29
|
jid: job_hash["jid"]
|
43
30
|
}
|
44
|
-
h[:bid] = job_hash["bid"] if job_hash
|
45
|
-
h[:tags] = job_hash["tags"] if job_hash
|
46
|
-
h
|
47
|
-
end
|
48
|
-
|
49
|
-
def with_elapsed_time_context(start, &block)
|
50
|
-
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
51
|
-
end
|
31
|
+
h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
|
32
|
+
h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
|
52
33
|
|
53
|
-
|
54
|
-
|
34
|
+
Thread.current[:sidekiq_context] = h
|
35
|
+
level = job_hash["log_level"]
|
36
|
+
if level
|
37
|
+
@logger.log_at(level, &block)
|
38
|
+
else
|
39
|
+
yield
|
40
|
+
end
|
41
|
+
ensure
|
42
|
+
Thread.current[:sidekiq_context] = nil
|
55
43
|
end
|
56
44
|
|
57
45
|
private
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -25,18 +25,19 @@ module Sidekiq
|
|
25
25
|
#
|
26
26
|
# A job looks like:
|
27
27
|
#
|
28
|
-
# { 'class' => '
|
28
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
|
29
29
|
#
|
30
30
|
# The 'retry' option also accepts a number (in place of 'true'):
|
31
31
|
#
|
32
|
-
# { 'class' => '
|
32
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
|
33
33
|
#
|
34
34
|
# The job will be retried this number of times before giving up. (If simply
|
35
35
|
# 'true', Sidekiq retries 25 times)
|
36
36
|
#
|
37
|
-
#
|
37
|
+
# Relevant options for job retries:
|
38
38
|
#
|
39
|
-
# * 'queue' - the queue
|
39
|
+
# * 'queue' - the queue for the initial job
|
40
|
+
# * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
|
40
41
|
# * 'retry_count' - number of times we've retried so far.
|
41
42
|
# * 'error_message' - the message from the exception
|
42
43
|
# * 'error_class' - the exception class
|
@@ -52,15 +53,17 @@ module Sidekiq
|
|
52
53
|
#
|
53
54
|
# Sidekiq.options[:max_retries] = 7
|
54
55
|
#
|
55
|
-
# or limit the number of retries for a particular
|
56
|
+
# or limit the number of retries for a particular job and send retries to
|
57
|
+
# a low priority queue with:
|
56
58
|
#
|
57
|
-
# class
|
58
|
-
# include Sidekiq::
|
59
|
-
# sidekiq_options :
|
59
|
+
# class MyJob
|
60
|
+
# include Sidekiq::Job
|
61
|
+
# sidekiq_options retry: 10, retry_queue: 'low'
|
60
62
|
# end
|
61
63
|
#
|
62
64
|
class JobRetry
|
63
65
|
class Handled < ::RuntimeError; end
|
66
|
+
|
64
67
|
class Skip < Handled; end
|
65
68
|
|
66
69
|
include Sidekiq::Util
|
@@ -73,7 +76,7 @@ module Sidekiq
|
|
73
76
|
|
74
77
|
# The global retry handler requires only the barest of data.
|
75
78
|
# We want to be able to retry as much as possible so we don't
|
76
|
-
# require the
|
79
|
+
# require the job to be instantiated.
|
77
80
|
def global(jobstr, queue)
|
78
81
|
yield
|
79
82
|
rescue Handled => ex
|
@@ -100,14 +103,14 @@ module Sidekiq
|
|
100
103
|
end
|
101
104
|
|
102
105
|
# The local retry support means that any errors that occur within
|
103
|
-
# this block can be associated with the given
|
106
|
+
# this block can be associated with the given job instance.
|
104
107
|
# This is required to support the `sidekiq_retries_exhausted` block.
|
105
108
|
#
|
106
109
|
# Note that any exception from the block is wrapped in the Skip
|
107
110
|
# exception so the global block does not reprocess the error. The
|
108
111
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
109
112
|
# calling the handle_exception handlers.
|
110
|
-
def local(
|
113
|
+
def local(jobinst, jobstr, queue)
|
111
114
|
yield
|
112
115
|
rescue Handled => ex
|
113
116
|
raise ex
|
@@ -120,11 +123,11 @@ module Sidekiq
|
|
120
123
|
|
121
124
|
msg = Sidekiq.load_json(jobstr)
|
122
125
|
if msg["retry"].nil?
|
123
|
-
msg["retry"] =
|
126
|
+
msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
|
124
127
|
end
|
125
128
|
|
126
129
|
raise e unless msg["retry"]
|
127
|
-
attempt_retry(
|
130
|
+
attempt_retry(jobinst, msg, queue, e)
|
128
131
|
# We've handled this error associated with this job, don't
|
129
132
|
# need to handle it at the global level
|
130
133
|
raise Skip
|
@@ -132,10 +135,10 @@ module Sidekiq
|
|
132
135
|
|
133
136
|
private
|
134
137
|
|
135
|
-
# Note that +
|
136
|
-
# instantiate the
|
138
|
+
# Note that +jobinst+ can be nil here if an error is raised before we can
|
139
|
+
# instantiate the job instance. All access must be guarded and
|
137
140
|
# best effort.
|
138
|
-
def attempt_retry(
|
141
|
+
def attempt_retry(jobinst, msg, queue, exception)
|
139
142
|
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
140
143
|
|
141
144
|
msg["queue"] = (msg["retry_queue"] || queue)
|
@@ -167,7 +170,7 @@ module Sidekiq
|
|
167
170
|
end
|
168
171
|
|
169
172
|
if count < max_retry_attempts
|
170
|
-
delay = delay_for(
|
173
|
+
delay = delay_for(jobinst, count, exception)
|
171
174
|
# Logging here can break retries if the logging device raises ENOSPC #3979
|
172
175
|
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
173
176
|
retry_at = Time.now.to_f + delay
|
@@ -177,13 +180,13 @@ module Sidekiq
|
|
177
180
|
end
|
178
181
|
else
|
179
182
|
# Goodbye dear message, you (re)tried your best I'm sure.
|
180
|
-
retries_exhausted(
|
183
|
+
retries_exhausted(jobinst, msg, exception)
|
181
184
|
end
|
182
185
|
end
|
183
186
|
|
184
|
-
def retries_exhausted(
|
187
|
+
def retries_exhausted(jobinst, msg, exception)
|
185
188
|
begin
|
186
|
-
block =
|
189
|
+
block = jobinst&.sidekiq_retries_exhausted_block
|
187
190
|
block&.call(msg, exception)
|
188
191
|
rescue => e
|
189
192
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
@@ -212,23 +215,19 @@ module Sidekiq
|
|
212
215
|
end
|
213
216
|
end
|
214
217
|
|
215
|
-
def delay_for(
|
216
|
-
|
217
|
-
|
218
|
-
|
218
|
+
def delay_for(jobinst, count, exception)
|
219
|
+
jitter = rand(10) * (count + 1)
|
220
|
+
if jobinst&.sidekiq_retry_in_block
|
221
|
+
custom_retry_in = retry_in(jobinst, count, exception).to_i
|
222
|
+
return custom_retry_in + jitter if custom_retry_in > 0
|
219
223
|
end
|
220
|
-
|
221
|
-
end
|
222
|
-
|
223
|
-
# delayed_job uses the same basic formula
|
224
|
-
def seconds_to_delay(count)
|
225
|
-
(count**4) + 15 + (rand(30) * (count + 1))
|
224
|
+
(count**4) + 15 + jitter
|
226
225
|
end
|
227
226
|
|
228
|
-
def retry_in(
|
229
|
-
|
227
|
+
def retry_in(jobinst, count, exception)
|
228
|
+
jobinst.sidekiq_retry_in_block.call(count, exception)
|
230
229
|
rescue Exception => e
|
231
|
-
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{
|
230
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
232
231
|
nil
|
233
232
|
end
|
234
233
|
|