sidekiq 6.4.0 → 7.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +232 -12
- data/README.md +44 -31
- data/bin/sidekiq +4 -9
- data/bin/sidekiqload +207 -117
- data/bin/sidekiqmon +4 -1
- data/lib/sidekiq/api.rb +329 -188
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +85 -81
- data/lib/sidekiq/client.rb +98 -58
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +278 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +23 -24
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +80 -56
- data/lib/sidekiq/job_util.rb +60 -20
- data/lib/sidekiq/launcher.rb +103 -95
- data/lib/sidekiq/logger.rb +9 -44
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +58 -20
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +22 -10
- data/lib/sidekiq/redis_client_adapter.rb +96 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +66 -38
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +40 -9
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +35 -21
- data/lib/sidekiq/web.rb +10 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -206
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +66 -297
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +44 -28
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +56 -27
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -364
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/scheduled"
|
4
|
-
require "sidekiq/api"
|
5
|
-
|
6
3
|
require "zlib"
|
7
4
|
require "base64"
|
5
|
+
require "sidekiq/component"
|
8
6
|
|
9
7
|
module Sidekiq
|
10
8
|
##
|
@@ -25,11 +23,11 @@ module Sidekiq
|
|
25
23
|
#
|
26
24
|
# A job looks like:
|
27
25
|
#
|
28
|
-
# { 'class' => '
|
26
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
|
29
27
|
#
|
30
28
|
# The 'retry' option also accepts a number (in place of 'true'):
|
31
29
|
#
|
32
|
-
# { 'class' => '
|
30
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
|
33
31
|
#
|
34
32
|
# The job will be retried this number of times before giving up. (If simply
|
35
33
|
# 'true', Sidekiq retries 25 times)
|
@@ -51,13 +49,13 @@ module Sidekiq
|
|
51
49
|
# The default number of retries is 25 which works out to about 3 weeks
|
52
50
|
# You can change the default maximum number of retries in your initializer:
|
53
51
|
#
|
54
|
-
# Sidekiq.
|
52
|
+
# Sidekiq.default_configuration[:max_retries] = 7
|
55
53
|
#
|
56
|
-
# or limit the number of retries for a particular
|
54
|
+
# or limit the number of retries for a particular job and send retries to
|
57
55
|
# a low priority queue with:
|
58
56
|
#
|
59
|
-
# class
|
60
|
-
# include Sidekiq::
|
57
|
+
# class MyJob
|
58
|
+
# include Sidekiq::Job
|
61
59
|
# sidekiq_options retry: 10, retry_queue: 'low'
|
62
60
|
# end
|
63
61
|
#
|
@@ -66,17 +64,19 @@ module Sidekiq
|
|
66
64
|
|
67
65
|
class Skip < Handled; end
|
68
66
|
|
69
|
-
include Sidekiq::
|
67
|
+
include Sidekiq::Component
|
70
68
|
|
71
69
|
DEFAULT_MAX_RETRY_ATTEMPTS = 25
|
72
70
|
|
73
|
-
def initialize(
|
74
|
-
@
|
71
|
+
def initialize(capsule)
|
72
|
+
@config = @capsule = capsule
|
73
|
+
@max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
|
74
|
+
@backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
|
75
75
|
end
|
76
76
|
|
77
77
|
# The global retry handler requires only the barest of data.
|
78
78
|
# We want to be able to retry as much as possible so we don't
|
79
|
-
# require the
|
79
|
+
# require the job to be instantiated.
|
80
80
|
def global(jobstr, queue)
|
81
81
|
yield
|
82
82
|
rescue Handled => ex
|
@@ -90,9 +90,9 @@ module Sidekiq
|
|
90
90
|
|
91
91
|
msg = Sidekiq.load_json(jobstr)
|
92
92
|
if msg["retry"]
|
93
|
-
|
93
|
+
process_retry(nil, msg, queue, e)
|
94
94
|
else
|
95
|
-
|
95
|
+
@capsule.config.death_handlers.each do |handler|
|
96
96
|
handler.call(msg, e)
|
97
97
|
rescue => handler_ex
|
98
98
|
handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
|
@@ -103,14 +103,14 @@ module Sidekiq
|
|
103
103
|
end
|
104
104
|
|
105
105
|
# The local retry support means that any errors that occur within
|
106
|
-
# this block can be associated with the given
|
106
|
+
# this block can be associated with the given job instance.
|
107
107
|
# This is required to support the `sidekiq_retries_exhausted` block.
|
108
108
|
#
|
109
109
|
# Note that any exception from the block is wrapped in the Skip
|
110
110
|
# exception so the global block does not reprocess the error. The
|
111
111
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
112
112
|
# calling the handle_exception handlers.
|
113
|
-
def local(
|
113
|
+
def local(jobinst, jobstr, queue)
|
114
114
|
yield
|
115
115
|
rescue Handled => ex
|
116
116
|
raise ex
|
@@ -123,11 +123,11 @@ module Sidekiq
|
|
123
123
|
|
124
124
|
msg = Sidekiq.load_json(jobstr)
|
125
125
|
if msg["retry"].nil?
|
126
|
-
msg["retry"] =
|
126
|
+
msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
|
127
127
|
end
|
128
128
|
|
129
129
|
raise e unless msg["retry"]
|
130
|
-
|
130
|
+
process_retry(jobinst, msg, queue, e)
|
131
131
|
# We've handled this error associated with this job, don't
|
132
132
|
# need to handle it at the global level
|
133
133
|
raise Skip
|
@@ -135,10 +135,10 @@ module Sidekiq
|
|
135
135
|
|
136
136
|
private
|
137
137
|
|
138
|
-
# Note that +
|
139
|
-
# instantiate the
|
138
|
+
# Note that +jobinst+ can be nil here if an error is raised before we can
|
139
|
+
# instantiate the job instance. All access must be guarded and
|
140
140
|
# best effort.
|
141
|
-
def
|
141
|
+
def process_retry(jobinst, msg, queue, exception)
|
142
142
|
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
143
143
|
|
144
144
|
msg["queue"] = (msg["retry_queue"] || queue)
|
@@ -160,33 +160,65 @@ module Sidekiq
|
|
160
160
|
end
|
161
161
|
|
162
162
|
if msg["backtrace"]
|
163
|
+
backtrace = @backtrace_cleaner.call(exception.backtrace)
|
163
164
|
lines = if msg["backtrace"] == true
|
164
|
-
|
165
|
+
backtrace
|
165
166
|
else
|
166
|
-
|
167
|
+
backtrace[0...msg["backtrace"].to_i]
|
167
168
|
end
|
168
169
|
|
169
170
|
msg["error_backtrace"] = compress_backtrace(lines)
|
170
171
|
end
|
171
172
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
173
|
+
# Goodbye dear message, you (re)tried your best I'm sure.
|
174
|
+
return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
|
175
|
+
|
176
|
+
strategy, delay = delay_for(jobinst, count, exception, msg)
|
177
|
+
case strategy
|
178
|
+
when :discard
|
179
|
+
return # poof!
|
180
|
+
when :kill
|
181
|
+
return retries_exhausted(jobinst, msg, exception)
|
182
|
+
end
|
183
|
+
|
184
|
+
# Logging here can break retries if the logging device raises ENOSPC #3979
|
185
|
+
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
186
|
+
jitter = rand(10) * (count + 1)
|
187
|
+
retry_at = Time.now.to_f + delay + jitter
|
188
|
+
payload = Sidekiq.dump_json(msg)
|
189
|
+
redis do |conn|
|
190
|
+
conn.zadd("retry", retry_at.to_s, payload)
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
# returns (strategy, seconds)
|
195
|
+
def delay_for(jobinst, count, exception, msg)
|
196
|
+
rv = begin
|
197
|
+
# sidekiq_retry_in can return two different things:
|
198
|
+
# 1. When to retry next, as an integer of seconds
|
199
|
+
# 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
|
200
|
+
jobinst&.sidekiq_retry_in_block&.call(count, exception, msg)
|
201
|
+
rescue Exception => e
|
202
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
203
|
+
nil
|
204
|
+
end
|
205
|
+
|
206
|
+
rv = rv.to_i if rv.respond_to?(:to_i)
|
207
|
+
delay = (count**4) + 15
|
208
|
+
if Integer === rv && rv > 0
|
209
|
+
delay = rv
|
210
|
+
elsif rv == :discard
|
211
|
+
return [:discard, nil] # do nothing, job goes poof
|
212
|
+
elsif rv == :kill
|
213
|
+
return [:kill, nil]
|
184
214
|
end
|
215
|
+
|
216
|
+
[:default, delay]
|
185
217
|
end
|
186
218
|
|
187
|
-
def retries_exhausted(
|
219
|
+
def retries_exhausted(jobinst, msg, exception)
|
188
220
|
begin
|
189
|
-
block =
|
221
|
+
block = jobinst&.sidekiq_retries_exhausted_block
|
190
222
|
block&.call(msg, exception)
|
191
223
|
rescue => e
|
192
224
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
@@ -194,7 +226,7 @@ module Sidekiq
|
|
194
226
|
|
195
227
|
send_to_morgue(msg) unless msg["dead"] == false
|
196
228
|
|
197
|
-
|
229
|
+
@capsule.config.death_handlers.each do |handler|
|
198
230
|
handler.call(msg, exception)
|
199
231
|
rescue => e
|
200
232
|
handle_exception(e, {context: "Error calling death handler", job: msg})
|
@@ -204,7 +236,15 @@ module Sidekiq
|
|
204
236
|
def send_to_morgue(msg)
|
205
237
|
logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
|
206
238
|
payload = Sidekiq.dump_json(msg)
|
207
|
-
|
239
|
+
now = Time.now.to_f
|
240
|
+
|
241
|
+
redis do |conn|
|
242
|
+
conn.multi do |xa|
|
243
|
+
xa.zadd("dead", now.to_s, payload)
|
244
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
245
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
246
|
+
end
|
247
|
+
end
|
208
248
|
end
|
209
249
|
|
210
250
|
def retry_attempts_from(msg_retry, default)
|
@@ -215,22 +255,6 @@ module Sidekiq
|
|
215
255
|
end
|
216
256
|
end
|
217
257
|
|
218
|
-
def delay_for(worker, count, exception)
|
219
|
-
jitter = rand(10) * (count + 1)
|
220
|
-
if worker&.sidekiq_retry_in_block
|
221
|
-
custom_retry_in = retry_in(worker, count, exception).to_i
|
222
|
-
return custom_retry_in + jitter if custom_retry_in > 0
|
223
|
-
end
|
224
|
-
(count**4) + 15 + jitter
|
225
|
-
end
|
226
|
-
|
227
|
-
def retry_in(worker, count, exception)
|
228
|
-
worker.sidekiq_retry_in_block.call(count, exception)
|
229
|
-
rescue Exception => e
|
230
|
-
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
|
231
|
-
nil
|
232
|
-
end
|
233
|
-
|
234
258
|
def exception_caused_by_shutdown?(e, checked_causes = [])
|
235
259
|
return false unless e.cause
|
236
260
|
|
data/lib/sidekiq/job_util.rb
CHANGED
@@ -4,27 +4,36 @@ require "time"
|
|
4
4
|
module Sidekiq
|
5
5
|
module JobUtil
|
6
6
|
# These functions encapsulate various job utilities.
|
7
|
-
|
7
|
+
|
8
|
+
TRANSIENT_ATTRIBUTES = %w[]
|
8
9
|
|
9
10
|
def validate(item)
|
10
11
|
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
11
|
-
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
12
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
|
12
13
|
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
13
14
|
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
14
15
|
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
16
|
+
end
|
17
|
+
|
18
|
+
def verify_json(item)
|
19
|
+
job_class = item["wrapped"] || item["class"]
|
20
|
+
args = item["args"]
|
21
|
+
mode = Sidekiq::Config::DEFAULTS[:on_complex_arguments]
|
15
22
|
|
16
|
-
if
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
23
|
+
if mode == :raise || mode == :warn
|
24
|
+
if (unsafe_item = json_unsafe?(args))
|
25
|
+
msg = <<~EOM
|
26
|
+
Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
|
27
|
+
See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
|
28
|
+
To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
|
29
|
+
EOM
|
30
|
+
|
31
|
+
if mode == :raise
|
32
|
+
raise(ArgumentError, msg)
|
33
|
+
else
|
34
|
+
warn(msg)
|
35
|
+
end
|
36
|
+
end
|
28
37
|
end
|
29
38
|
end
|
30
39
|
|
@@ -39,27 +48,58 @@ module Sidekiq
|
|
39
48
|
|
40
49
|
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
41
50
|
|
51
|
+
# remove job attributes which aren't necessary to persist into Redis
|
52
|
+
TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
|
53
|
+
|
54
|
+
item["jid"] ||= SecureRandom.hex(12)
|
42
55
|
item["class"] = item["class"].to_s
|
43
56
|
item["queue"] = item["queue"].to_s
|
44
|
-
item["jid"] ||= SecureRandom.hex(12)
|
45
57
|
item["created_at"] ||= Time.now.to_f
|
46
|
-
|
47
58
|
item
|
48
59
|
end
|
49
60
|
|
50
61
|
def normalized_hash(item_class)
|
51
62
|
if item_class.is_a?(Class)
|
52
|
-
raise(ArgumentError, "Message must include a Sidekiq::
|
63
|
+
raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
|
53
64
|
item_class.get_sidekiq_options
|
54
65
|
else
|
55
|
-
Sidekiq.
|
66
|
+
Sidekiq.default_job_options
|
56
67
|
end
|
57
68
|
end
|
58
69
|
|
59
70
|
private
|
60
71
|
|
61
|
-
|
62
|
-
|
72
|
+
RECURSIVE_JSON_UNSAFE = {
|
73
|
+
Integer => ->(val) {},
|
74
|
+
Float => ->(val) {},
|
75
|
+
TrueClass => ->(val) {},
|
76
|
+
FalseClass => ->(val) {},
|
77
|
+
NilClass => ->(val) {},
|
78
|
+
String => ->(val) {},
|
79
|
+
Array => ->(val) {
|
80
|
+
val.each do |e|
|
81
|
+
unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e)
|
82
|
+
return unsafe_item unless unsafe_item.nil?
|
83
|
+
end
|
84
|
+
nil
|
85
|
+
},
|
86
|
+
Hash => ->(val) {
|
87
|
+
val.each do |k, v|
|
88
|
+
return k unless String === k
|
89
|
+
|
90
|
+
unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v)
|
91
|
+
return unsafe_item unless unsafe_item.nil?
|
92
|
+
end
|
93
|
+
nil
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
|
98
|
+
RECURSIVE_JSON_UNSAFE.compare_by_identity
|
99
|
+
private_constant :RECURSIVE_JSON_UNSAFE
|
100
|
+
|
101
|
+
def json_unsafe?(item)
|
102
|
+
RECURSIVE_JSON_UNSAFE[item.class].call(item)
|
63
103
|
end
|
64
104
|
end
|
65
105
|
end
|