sidekiq 6.0.0 → 6.3.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +258 -2
- data/LICENSE +1 -1
- data/README.md +6 -8
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +11 -1
- data/lib/sidekiq/api.rb +220 -145
- data/lib/sidekiq/cli.rb +64 -27
- data/lib/sidekiq/client.rb +31 -14
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -1
- data/lib/sidekiq/fetch.rb +36 -27
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +13 -5
- data/lib/sidekiq/job_retry.rb +27 -17
- data/lib/sidekiq/launcher.rb +110 -28
- data/lib/sidekiq/logger.rb +109 -12
- data/lib/sidekiq/manager.rb +4 -4
- data/lib/sidekiq/middleware/chain.rb +17 -6
- data/lib/sidekiq/middleware/current_attributes.rb +48 -0
- data/lib/sidekiq/monitor.rb +3 -18
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +22 -24
- data/lib/sidekiq/rails.rb +27 -18
- data/lib/sidekiq/redis_connection.rb +19 -13
- data/lib/sidekiq/scheduled.rb +37 -11
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +14 -4
- data/lib/sidekiq/util.rb +28 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +37 -30
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +51 -33
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +37 -73
- data/lib/sidekiq/worker.rb +78 -14
- data/lib/sidekiq.rb +24 -8
- data/sidekiq.gemspec +13 -6
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +53 -53
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +43 -230
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +6 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +5 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +19 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +29 -50
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
data/lib/sidekiq/cli.rb
CHANGED
@@ -33,17 +33,29 @@ module Sidekiq
|
|
33
33
|
# Code within this method is not tested because it alters
|
34
34
|
# global process state irreversibly. PRs which improve the
|
35
35
|
# test coverage of Sidekiq::CLI are welcomed.
|
36
|
-
def run
|
37
|
-
|
36
|
+
def run(boot_app: true)
|
37
|
+
boot_application if boot_app
|
38
|
+
|
38
39
|
if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
39
40
|
print_banner
|
40
41
|
end
|
42
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
41
43
|
|
42
44
|
self_read, self_write = IO.pipe
|
43
45
|
sigs = %w[INT TERM TTIN TSTP]
|
46
|
+
# USR1 and USR2 don't work on the JVM
|
47
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
44
48
|
sigs.each do |sig|
|
45
|
-
trap
|
46
|
-
|
49
|
+
old_handler = Signal.trap(sig) do
|
50
|
+
if old_handler.respond_to?(:call)
|
51
|
+
begin
|
52
|
+
old_handler.call
|
53
|
+
rescue Exception => exc
|
54
|
+
# signal handlers can't use Logger so puts only
|
55
|
+
puts ["Error in #{sig} handler", exc].inspect
|
56
|
+
end
|
57
|
+
end
|
58
|
+
self_write.puts(sig)
|
47
59
|
end
|
48
60
|
rescue ArgumentError
|
49
61
|
puts "Signal #{sig} not supported"
|
@@ -51,12 +63,25 @@ module Sidekiq
|
|
51
63
|
|
52
64
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
53
65
|
logger.info Sidekiq::LICENSE
|
54
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
66
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
55
67
|
|
56
68
|
# touch the connection pool so it is created before we
|
57
69
|
# fire startup and start multithreading.
|
58
|
-
|
59
|
-
|
70
|
+
info = Sidekiq.redis_info
|
71
|
+
ver = info["redis_version"]
|
72
|
+
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
73
|
+
|
74
|
+
maxmemory_policy = info["maxmemory_policy"]
|
75
|
+
if maxmemory_policy != "noeviction"
|
76
|
+
logger.warn <<~EOM
|
77
|
+
|
78
|
+
|
79
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
80
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
81
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
82
|
+
|
83
|
+
EOM
|
84
|
+
end
|
60
85
|
|
61
86
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
62
87
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
@@ -160,17 +185,14 @@ module Sidekiq
|
|
160
185
|
Sidekiq.logger.warn "<no backtrace available>"
|
161
186
|
end
|
162
187
|
end
|
163
|
-
}
|
188
|
+
}
|
164
189
|
}
|
190
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
|
191
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
165
192
|
|
166
193
|
def handle_signal(sig)
|
167
194
|
Sidekiq.logger.debug "Got #{sig} signal"
|
168
|
-
|
169
|
-
if handy
|
170
|
-
handy.call(self)
|
171
|
-
else
|
172
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
173
|
-
end
|
195
|
+
SIGNAL_HANDLERS[sig].call(self)
|
174
196
|
end
|
175
197
|
|
176
198
|
private
|
@@ -182,7 +204,11 @@ module Sidekiq
|
|
182
204
|
end
|
183
205
|
|
184
206
|
def set_environment(cli_env)
|
185
|
-
|
207
|
+
# See #984 for discussion.
|
208
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
209
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
210
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
211
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
186
212
|
end
|
187
213
|
|
188
214
|
def symbolize_keys_deep!(hash)
|
@@ -204,7 +230,7 @@ module Sidekiq
|
|
204
230
|
|
205
231
|
# check config file presence
|
206
232
|
if opts[:config_file]
|
207
|
-
|
233
|
+
unless File.exist?(opts[:config_file])
|
208
234
|
raise ArgumentError, "No such file #{opts[:config_file]}"
|
209
235
|
end
|
210
236
|
else
|
@@ -224,8 +250,7 @@ module Sidekiq
|
|
224
250
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
225
251
|
|
226
252
|
# set defaults
|
227
|
-
opts[:queues] =
|
228
|
-
opts[:strict] = true if opts[:strict].nil?
|
253
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
229
254
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
230
255
|
|
231
256
|
# merge with defaults
|
@@ -236,7 +261,7 @@ module Sidekiq
|
|
236
261
|
Sidekiq.options
|
237
262
|
end
|
238
263
|
|
239
|
-
def
|
264
|
+
def boot_application
|
240
265
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
241
266
|
|
242
267
|
if File.directory?(options[:require])
|
@@ -283,8 +308,13 @@ module Sidekiq
|
|
283
308
|
|
284
309
|
def parse_options(argv)
|
285
310
|
opts = {}
|
311
|
+
@parser = option_parser(opts)
|
312
|
+
@parser.parse!(argv)
|
313
|
+
opts
|
314
|
+
end
|
286
315
|
|
287
|
-
|
316
|
+
def option_parser(opts)
|
317
|
+
parser = OptionParser.new { |o|
|
288
318
|
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
289
319
|
opts[:concurrency] = Integer(arg)
|
290
320
|
end
|
@@ -336,15 +366,13 @@ module Sidekiq
|
|
336
366
|
end
|
337
367
|
}
|
338
368
|
|
339
|
-
|
340
|
-
|
341
|
-
logger.info
|
369
|
+
parser.banner = "sidekiq [options]"
|
370
|
+
parser.on_tail "-h", "--help", "Show help" do
|
371
|
+
logger.info parser
|
342
372
|
die 1
|
343
373
|
end
|
344
374
|
|
345
|
-
|
346
|
-
|
347
|
-
opts
|
375
|
+
parser
|
348
376
|
end
|
349
377
|
|
350
378
|
def initialize_logger
|
@@ -361,6 +389,8 @@ module Sidekiq
|
|
361
389
|
end
|
362
390
|
|
363
391
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
392
|
+
opts.delete(:strict)
|
393
|
+
|
364
394
|
parse_queues(opts, opts.delete(:queues) || [])
|
365
395
|
|
366
396
|
opts
|
@@ -372,9 +402,16 @@ module Sidekiq
|
|
372
402
|
|
373
403
|
def parse_queue(opts, queue, weight = nil)
|
374
404
|
opts[:queues] ||= []
|
405
|
+
opts[:strict] = true if opts[:strict].nil?
|
375
406
|
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
376
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue }
|
407
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
377
408
|
opts[:strict] = false if weight.to_i > 0
|
378
409
|
end
|
410
|
+
|
411
|
+
def rails_app?
|
412
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
413
|
+
end
|
379
414
|
end
|
380
415
|
end
|
416
|
+
|
417
|
+
require "sidekiq/systemd"
|
data/lib/sidekiq/client.rb
CHANGED
@@ -19,7 +19,7 @@ module Sidekiq
|
|
19
19
|
#
|
20
20
|
def middleware(&block)
|
21
21
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
22
|
+
if block
|
23
23
|
@chain = @chain.dup
|
24
24
|
yield @chain
|
25
25
|
end
|
@@ -90,13 +90,19 @@ module Sidekiq
|
|
90
90
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
91
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
92
|
def push_bulk(items)
|
93
|
-
|
94
|
-
|
95
|
-
|
93
|
+
args = items["args"]
|
94
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
95
|
+
return [] if args.empty? # no jobs to push
|
96
|
+
|
97
|
+
at = items.delete("at")
|
98
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
99
|
+
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
96
100
|
|
97
101
|
normed = normalize_item(items)
|
98
|
-
payloads =
|
99
|
-
copy = normed.merge("args" =>
|
102
|
+
payloads = args.map.with_index { |job_args, index|
|
103
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
|
104
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
105
|
+
|
100
106
|
result = process_single(items["class"], copy)
|
101
107
|
result || nil
|
102
108
|
}.compact
|
@@ -180,7 +186,7 @@ module Sidekiq
|
|
180
186
|
|
181
187
|
def raw_push(payloads)
|
182
188
|
@redis_pool.with do |conn|
|
183
|
-
conn.
|
189
|
+
conn.pipelined do
|
184
190
|
atomic_push(conn, payloads)
|
185
191
|
end
|
186
192
|
end
|
@@ -188,7 +194,7 @@ module Sidekiq
|
|
188
194
|
end
|
189
195
|
|
190
196
|
def atomic_push(conn, payloads)
|
191
|
-
if payloads.first
|
197
|
+
if payloads.first.key?("at")
|
192
198
|
conn.zadd("schedule", payloads.map { |hash|
|
193
199
|
at = hash.delete("at").to_s
|
194
200
|
[at, Sidekiq.dump_json(hash)]
|
@@ -213,20 +219,31 @@ module Sidekiq
|
|
213
219
|
end
|
214
220
|
end
|
215
221
|
|
222
|
+
def validate(item)
|
223
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
224
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
225
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
226
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
227
|
+
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
228
|
+
end
|
229
|
+
|
216
230
|
def normalize_item(item)
|
217
|
-
|
218
|
-
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
219
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
220
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
231
|
+
validate(item)
|
221
232
|
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
222
233
|
|
223
|
-
|
224
|
-
|
234
|
+
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
235
|
+
# this allows ActiveJobs to control sidekiq_options too.
|
236
|
+
defaults = normalized_hash(item["class"])
|
237
|
+
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
238
|
+
item = defaults.merge(item)
|
239
|
+
|
240
|
+
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
225
241
|
|
226
242
|
item["class"] = item["class"].to_s
|
227
243
|
item["queue"] = item["queue"].to_s
|
228
244
|
item["jid"] ||= SecureRandom.hex(12)
|
229
245
|
item["created_at"] ||= Time.now.to_f
|
246
|
+
|
230
247
|
item
|
231
248
|
end
|
232
249
|
|
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
10
10
|
#
|
11
|
+
# @example
|
11
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
12
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
13
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
12
13
|
#
|
13
14
|
# Please note, this is not recommended as this will serialize the entire
|
14
15
|
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
12
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.delay.delete_inactive
|
13
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
13
14
|
#
|
14
15
|
class DelayedClass
|
15
16
|
include Sidekiq::Worker
|
@@ -24,7 +24,9 @@ module Sidekiq
|
|
24
24
|
if marshalled.size > SIZE_LIMIT
|
25
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
26
26
|
end
|
27
|
-
@performable.client_push({"class" => @performable,
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -14,61 +14,54 @@ module Sidekiq
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def queue_name
|
17
|
-
queue.
|
17
|
+
queue.delete_prefix("queue:")
|
18
18
|
end
|
19
19
|
|
20
20
|
def requeue
|
21
21
|
Sidekiq.redis do |conn|
|
22
|
-
conn.rpush(
|
22
|
+
conn.rpush(queue, job)
|
23
23
|
end
|
24
24
|
end
|
25
25
|
}
|
26
26
|
|
27
27
|
def initialize(options)
|
28
|
-
|
29
|
-
@
|
28
|
+
raise ArgumentError, "missing queue list" unless options[:queues]
|
29
|
+
@options = options
|
30
|
+
@strictly_ordered_queues = !!@options[:strict]
|
31
|
+
@queues = @options[:queues].map { |q| "queue:#{q}" }
|
30
32
|
if @strictly_ordered_queues
|
31
|
-
@queues
|
33
|
+
@queues.uniq!
|
32
34
|
@queues << TIMEOUT
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
36
38
|
def retrieve_work
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
# data from the first queue that has pending elements. We
|
44
|
-
# recreate the queue command each time we invoke Redis#brpop
|
45
|
-
# to honor weights and avoid queue starvation.
|
46
|
-
def queues_cmd
|
47
|
-
if @strictly_ordered_queues
|
48
|
-
@queues
|
49
|
-
else
|
50
|
-
queues = @queues.shuffle.uniq
|
51
|
-
queues << TIMEOUT
|
52
|
-
queues
|
39
|
+
qs = queues_cmd
|
40
|
+
# 4825 Sidekiq Pro with all queues paused will return an
|
41
|
+
# empty set of queues with a trailing TIMEOUT value.
|
42
|
+
if qs.size <= 1
|
43
|
+
sleep(TIMEOUT)
|
44
|
+
return nil
|
53
45
|
end
|
46
|
+
|
47
|
+
work = Sidekiq.redis { |conn| conn.brpop(*qs) }
|
48
|
+
UnitOfWork.new(*work) if work
|
54
49
|
end
|
55
50
|
|
56
|
-
|
57
|
-
# an instance method will make it async to the Fetcher actor
|
58
|
-
def self.bulk_requeue(inprogress, options)
|
51
|
+
def bulk_requeue(inprogress, options)
|
59
52
|
return if inprogress.empty?
|
60
53
|
|
61
54
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
55
|
jobs_to_requeue = {}
|
63
56
|
inprogress.each do |unit_of_work|
|
64
|
-
jobs_to_requeue[unit_of_work.
|
65
|
-
jobs_to_requeue[unit_of_work.
|
57
|
+
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
|
+
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
66
59
|
end
|
67
60
|
|
68
61
|
Sidekiq.redis do |conn|
|
69
62
|
conn.pipelined do
|
70
63
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
conn.rpush(
|
64
|
+
conn.rpush(queue, jobs)
|
72
65
|
end
|
73
66
|
end
|
74
67
|
end
|
@@ -76,5 +69,21 @@ module Sidekiq
|
|
76
69
|
rescue => ex
|
77
70
|
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
78
71
|
end
|
72
|
+
|
73
|
+
# Creating the Redis#brpop command takes into account any
|
74
|
+
# configured queue weights. By default Redis#brpop returns
|
75
|
+
# data from the first queue that has pending elements. We
|
76
|
+
# recreate the queue command each time we invoke Redis#brpop
|
77
|
+
# to honor weights and avoid queue starvation.
|
78
|
+
def queues_cmd
|
79
|
+
if @strictly_ordered_queues
|
80
|
+
@queues
|
81
|
+
else
|
82
|
+
permute = @queues.shuffle
|
83
|
+
permute.uniq!
|
84
|
+
permute << TIMEOUT
|
85
|
+
permute
|
86
|
+
end
|
87
|
+
end
|
79
88
|
end
|
80
89
|
end
|
data/lib/sidekiq/job.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "sidekiq/worker"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
|
5
|
+
# Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
|
6
|
+
#
|
7
|
+
# The term "worker" is too generic and overly confusing, used in several
|
8
|
+
# different contexts meaning different things. Many people call a Sidekiq
|
9
|
+
# process a "worker". Some people call the thread that executes jobs a
|
10
|
+
# "worker". This change brings Sidekiq closer to ActiveJob where your job
|
11
|
+
# classes extend ApplicationJob.
|
12
|
+
Job = Worker
|
13
|
+
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -23,23 +23,31 @@ module Sidekiq
|
|
23
23
|
raise
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
|
26
|
+
def prepare(job_hash, &block)
|
27
|
+
level = job_hash["log_level"]
|
28
|
+
if level
|
29
|
+
@logger.log_at(level) do
|
30
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
+
end
|
28
35
|
end
|
29
36
|
|
30
37
|
def job_hash_context(job_hash)
|
31
38
|
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
32
39
|
# attribute to expose the underlying thing.
|
33
40
|
h = {
|
34
|
-
class: job_hash["wrapped"] || job_hash["class"],
|
35
|
-
jid: job_hash["jid"]
|
41
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
|
42
|
+
jid: job_hash["jid"]
|
36
43
|
}
|
37
44
|
h[:bid] = job_hash["bid"] if job_hash["bid"]
|
45
|
+
h[:tags] = job_hash["tags"] if job_hash["tags"]
|
38
46
|
h
|
39
47
|
end
|
40
48
|
|
41
49
|
def with_elapsed_time_context(start, &block)
|
42
|
-
|
50
|
+
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
43
51
|
end
|
44
52
|
|
45
53
|
def elapsed_time_context(start)
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -3,6 +3,9 @@
|
|
3
3
|
require "sidekiq/scheduled"
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
|
+
require "zlib"
|
7
|
+
require "base64"
|
8
|
+
|
6
9
|
module Sidekiq
|
7
10
|
##
|
8
11
|
# Automatically retry jobs that fail in Sidekiq.
|
@@ -58,6 +61,7 @@ module Sidekiq
|
|
58
61
|
#
|
59
62
|
class JobRetry
|
60
63
|
class Handled < ::RuntimeError; end
|
64
|
+
|
61
65
|
class Skip < Handled; end
|
62
66
|
|
63
67
|
include Sidekiq::Util
|
@@ -71,7 +75,7 @@ module Sidekiq
|
|
71
75
|
# The global retry handler requires only the barest of data.
|
72
76
|
# We want to be able to retry as much as possible so we don't
|
73
77
|
# require the worker to be instantiated.
|
74
|
-
def global(
|
78
|
+
def global(jobstr, queue)
|
75
79
|
yield
|
76
80
|
rescue Handled => ex
|
77
81
|
raise ex
|
@@ -82,6 +86,7 @@ module Sidekiq
|
|
82
86
|
# ignore, will be pushed back onto queue during hard_shutdown
|
83
87
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
84
88
|
|
89
|
+
msg = Sidekiq.load_json(jobstr)
|
85
90
|
if msg["retry"]
|
86
91
|
attempt_retry(nil, msg, queue, e)
|
87
92
|
else
|
@@ -103,7 +108,7 @@ module Sidekiq
|
|
103
108
|
# exception so the global block does not reprocess the error. The
|
104
109
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
105
110
|
# calling the handle_exception handlers.
|
106
|
-
def local(worker,
|
111
|
+
def local(worker, jobstr, queue)
|
107
112
|
yield
|
108
113
|
rescue Handled => ex
|
109
114
|
raise ex
|
@@ -114,6 +119,7 @@ module Sidekiq
|
|
114
119
|
# ignore, will be pushed back onto queue during hard_shutdown
|
115
120
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
116
121
|
|
122
|
+
msg = Sidekiq.load_json(jobstr)
|
117
123
|
if msg["retry"].nil?
|
118
124
|
msg["retry"] = worker.class.get_sidekiq_options["retry"]
|
119
125
|
end
|
@@ -151,12 +157,14 @@ module Sidekiq
|
|
151
157
|
msg["retry_count"] = 0
|
152
158
|
end
|
153
159
|
|
154
|
-
if msg["backtrace"]
|
155
|
-
msg["
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
+
if msg["backtrace"]
|
161
|
+
lines = if msg["backtrace"] == true
|
162
|
+
exception.backtrace
|
163
|
+
else
|
164
|
+
exception.backtrace[0...msg["backtrace"].to_i]
|
165
|
+
end
|
166
|
+
|
167
|
+
msg["error_backtrace"] = compress_backtrace(lines)
|
160
168
|
end
|
161
169
|
|
162
170
|
if count < max_retry_attempts
|
@@ -182,13 +190,13 @@ module Sidekiq
|
|
182
190
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
183
191
|
end
|
184
192
|
|
193
|
+
send_to_morgue(msg) unless msg["dead"] == false
|
194
|
+
|
185
195
|
Sidekiq.death_handlers.each do |handler|
|
186
196
|
handler.call(msg, exception)
|
187
197
|
rescue => e
|
188
198
|
handle_exception(e, {context: "Error calling death handler", job: msg})
|
189
199
|
end
|
190
|
-
|
191
|
-
send_to_morgue(msg) unless msg["dead"] == false
|
192
200
|
end
|
193
201
|
|
194
202
|
def send_to_morgue(msg)
|
@@ -206,16 +214,12 @@ module Sidekiq
|
|
206
214
|
end
|
207
215
|
|
208
216
|
def delay_for(worker, count, exception)
|
217
|
+
jitter = rand(10) * (count + 1)
|
209
218
|
if worker&.sidekiq_retry_in_block
|
210
219
|
custom_retry_in = retry_in(worker, count, exception).to_i
|
211
|
-
return custom_retry_in if custom_retry_in > 0
|
220
|
+
return custom_retry_in + jitter if custom_retry_in > 0
|
212
221
|
end
|
213
|
-
|
214
|
-
end
|
215
|
-
|
216
|
-
# delayed_job uses the same basic formula
|
217
|
-
def seconds_to_delay(count)
|
218
|
-
(count**4) + 15 + (rand(30) * (count + 1))
|
222
|
+
(count**4) + 15 + jitter
|
219
223
|
end
|
220
224
|
|
221
225
|
def retry_in(worker, count, exception)
|
@@ -245,5 +249,11 @@ module Sidekiq
|
|
245
249
|
rescue
|
246
250
|
+"!!! ERROR MESSAGE THREW AN ERROR !!!"
|
247
251
|
end
|
252
|
+
|
253
|
+
def compress_backtrace(backtrace)
|
254
|
+
serialized = Sidekiq.dump_json(backtrace)
|
255
|
+
compressed = Zlib::Deflate.deflate(serialized)
|
256
|
+
Base64.encode64(compressed)
|
257
|
+
end
|
248
258
|
end
|
249
259
|
end
|