sidekiq 6.3.1 → 6.5.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +134 -0
- data/LICENSE +3 -3
- data/README.md +7 -2
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +261 -104
- data/lib/sidekiq/cli.rb +62 -38
- data/lib/sidekiq/client.rb +47 -67
- data/lib/sidekiq/{util.rb → component.rb} +12 -42
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
- data/lib/sidekiq/fetch.rb +20 -18
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +78 -55
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +58 -54
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +19 -8
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -49
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +66 -27
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +26 -7
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +21 -8
- data/lib/sidekiq/web.rb +8 -4
- data/lib/sidekiq/worker.rb +78 -19
- data/lib/sidekiq.rb +111 -30
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +13 -17
- data/web/assets/stylesheets/application.css +48 -6
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/ja.yml +7 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +9 -4
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +39 -13
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,34 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
12
13
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
14
|
|
15
|
-
module
|
15
|
+
# module ScoutApm
|
16
|
+
# VERSION = "5.3.1"
|
17
|
+
# end
|
18
|
+
fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
|
19
|
+
|
20
|
+
|
21
|
+
scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
|
22
|
+
|
23
|
+
|
24
|
+
EOM
|
25
|
+
|
26
|
+
module Sidekiq # :nodoc:
|
16
27
|
class CLI
|
17
|
-
include
|
28
|
+
include Sidekiq::Component
|
18
29
|
include Singleton unless $TESTING
|
19
30
|
|
20
31
|
attr_accessor :launcher
|
21
32
|
attr_accessor :environment
|
33
|
+
attr_accessor :config
|
34
|
+
|
35
|
+
def parse(args = ARGV.dup)
|
36
|
+
@config = Sidekiq
|
37
|
+
@config[:error_handlers].clear
|
38
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
22
39
|
|
23
|
-
def parse(args = ARGV)
|
24
40
|
setup_options(args)
|
25
41
|
initialize_logger
|
26
42
|
validate!
|
@@ -36,7 +52,7 @@ module Sidekiq
|
|
36
52
|
def run(boot_app: true)
|
37
53
|
boot_application if boot_app
|
38
54
|
|
39
|
-
if environment == "development" && $stdout.tty? &&
|
55
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
40
56
|
print_banner
|
41
57
|
end
|
42
58
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -67,7 +83,7 @@ module Sidekiq
|
|
67
83
|
|
68
84
|
# touch the connection pool so it is created before we
|
69
85
|
# fire startup and start multithreading.
|
70
|
-
info =
|
86
|
+
info = @config.redis_info
|
71
87
|
ver = info["redis_version"]
|
72
88
|
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
73
89
|
|
@@ -85,22 +101,22 @@ module Sidekiq
|
|
85
101
|
|
86
102
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
87
103
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
88
|
-
cursize =
|
89
|
-
needed =
|
104
|
+
cursize = @config.redis_pool.size
|
105
|
+
needed = @config[:concurrency] + 2
|
90
106
|
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
91
107
|
|
92
108
|
# cache process identity
|
93
|
-
|
109
|
+
@config[:identity] = identity
|
94
110
|
|
95
111
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
96
|
-
|
112
|
+
@config.server_middleware
|
97
113
|
|
98
114
|
# Before this point, the process is initializing with just the main thread.
|
99
115
|
# Starting here the process will now have multiple threads running.
|
100
116
|
fire_event(:startup, reverse: false, reraise: true)
|
101
117
|
|
102
|
-
logger.debug { "Client Middleware: #{
|
103
|
-
logger.debug { "Server Middleware: #{
|
118
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
119
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
104
120
|
|
105
121
|
launch(self_read)
|
106
122
|
end
|
@@ -110,13 +126,13 @@ module Sidekiq
|
|
110
126
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
111
127
|
end
|
112
128
|
|
113
|
-
@launcher = Sidekiq::Launcher.new(
|
129
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
114
130
|
|
115
131
|
begin
|
116
132
|
launcher.run
|
117
133
|
|
118
|
-
while
|
119
|
-
signal =
|
134
|
+
while self_read.wait_readable
|
135
|
+
signal = self_read.gets.strip
|
120
136
|
handle_signal(signal)
|
121
137
|
end
|
122
138
|
rescue Interrupt
|
@@ -173,25 +189,25 @@ module Sidekiq
|
|
173
189
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
174
190
|
"TERM" => ->(cli) { raise Interrupt },
|
175
191
|
"TSTP" => ->(cli) {
|
176
|
-
|
192
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
177
193
|
cli.launcher.quiet
|
178
194
|
},
|
179
195
|
"TTIN" => ->(cli) {
|
180
196
|
Thread.list.each do |thread|
|
181
|
-
|
197
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
182
198
|
if thread.backtrace
|
183
|
-
|
199
|
+
cli.logger.warn thread.backtrace.join("\n")
|
184
200
|
else
|
185
|
-
|
201
|
+
cli.logger.warn "<no backtrace available>"
|
186
202
|
end
|
187
203
|
end
|
188
204
|
}
|
189
205
|
}
|
190
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
206
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
191
207
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
192
208
|
|
193
209
|
def handle_signal(sig)
|
194
|
-
|
210
|
+
logger.debug "Got #{sig} signal"
|
195
211
|
SIGNAL_HANDLERS[sig].call(self)
|
196
212
|
end
|
197
213
|
|
@@ -209,6 +225,7 @@ module Sidekiq
|
|
209
225
|
# Both Sinatra 2.0+ and Sidekiq support this term.
|
210
226
|
# RAILS_ENV and RACK_ENV are there for legacy support.
|
211
227
|
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
228
|
+
config[:environment] = @environment
|
212
229
|
end
|
213
230
|
|
214
231
|
def symbolize_keys_deep!(hash)
|
@@ -237,7 +254,7 @@ module Sidekiq
|
|
237
254
|
config_dir = if File.directory?(opts[:require].to_s)
|
238
255
|
File.join(opts[:require], "config")
|
239
256
|
else
|
240
|
-
File.join(
|
257
|
+
File.join(@config[:require], "config")
|
241
258
|
end
|
242
259
|
|
243
260
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -254,27 +271,23 @@ module Sidekiq
|
|
254
271
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
255
272
|
|
256
273
|
# merge with defaults
|
257
|
-
|
258
|
-
end
|
259
|
-
|
260
|
-
def options
|
261
|
-
Sidekiq.options
|
274
|
+
@config.merge!(opts)
|
262
275
|
end
|
263
276
|
|
264
277
|
def boot_application
|
265
278
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
266
279
|
|
267
|
-
if File.directory?(
|
280
|
+
if File.directory?(@config[:require])
|
268
281
|
require "rails"
|
269
282
|
if ::Rails::VERSION::MAJOR < 5
|
270
283
|
raise "Sidekiq no longer supports this version of Rails"
|
271
284
|
else
|
272
285
|
require "sidekiq/rails"
|
273
|
-
require File.expand_path("#{
|
286
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
274
287
|
end
|
275
|
-
|
288
|
+
@config[:tag] ||= default_tag
|
276
289
|
else
|
277
|
-
require
|
290
|
+
require @config[:require]
|
278
291
|
end
|
279
292
|
end
|
280
293
|
|
@@ -291,18 +304,18 @@ module Sidekiq
|
|
291
304
|
end
|
292
305
|
|
293
306
|
def validate!
|
294
|
-
if !File.exist?(
|
295
|
-
(File.directory?(
|
307
|
+
if !File.exist?(@config[:require]) ||
|
308
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
296
309
|
logger.info "=================================================================="
|
297
310
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
298
|
-
logger.info " to load your
|
311
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
299
312
|
logger.info "=================================================================="
|
300
313
|
logger.info @parser
|
301
314
|
die(1)
|
302
315
|
end
|
303
316
|
|
304
317
|
[:concurrency, :timeout].each do |opt|
|
305
|
-
raise ArgumentError, "#{opt}: #{
|
318
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
306
319
|
end
|
307
320
|
end
|
308
321
|
|
@@ -336,7 +349,7 @@ module Sidekiq
|
|
336
349
|
parse_queue opts, queue, weight
|
337
350
|
end
|
338
351
|
|
339
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
352
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
340
353
|
opts[:require] = arg
|
341
354
|
end
|
342
355
|
|
@@ -376,11 +389,13 @@ module Sidekiq
|
|
376
389
|
end
|
377
390
|
|
378
391
|
def initialize_logger
|
379
|
-
|
392
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
380
393
|
end
|
381
394
|
|
382
395
|
def parse_config(path)
|
383
|
-
|
396
|
+
erb = ERB.new(File.read(path))
|
397
|
+
erb.filename = File.expand_path(path)
|
398
|
+
opts = load_yaml(erb.result) || {}
|
384
399
|
|
385
400
|
if opts.respond_to? :deep_symbolize_keys!
|
386
401
|
opts.deep_symbolize_keys!
|
@@ -396,6 +411,14 @@ module Sidekiq
|
|
396
411
|
opts
|
397
412
|
end
|
398
413
|
|
414
|
+
def load_yaml(src)
|
415
|
+
if Psych::VERSION > "4.0"
|
416
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
417
|
+
else
|
418
|
+
YAML.load(src)
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
399
422
|
def parse_queues(opts, queues_and_weights)
|
400
423
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
401
424
|
end
|
@@ -415,3 +438,4 @@ module Sidekiq
|
|
415
438
|
end
|
416
439
|
|
417
440
|
require "sidekiq/systemd"
|
441
|
+
require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
|
data/lib/sidekiq/client.rb
CHANGED
@@ -2,9 +2,12 @@
|
|
2
2
|
|
3
3
|
require "securerandom"
|
4
4
|
require "sidekiq/middleware/chain"
|
5
|
+
require "sidekiq/job_util"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
class Client
|
9
|
+
include Sidekiq::JobUtil
|
10
|
+
|
8
11
|
##
|
9
12
|
# Define client-side middleware:
|
10
13
|
#
|
@@ -12,7 +15,7 @@ module Sidekiq
|
|
12
15
|
# client.middleware do |chain|
|
13
16
|
# chain.use MyClientMiddleware
|
14
17
|
# end
|
15
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
16
19
|
#
|
17
20
|
# All client instances default to the globally-defined
|
18
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
@@ -46,16 +49,16 @@ module Sidekiq
|
|
46
49
|
# The main method used to push a job to Redis. Accepts a number of options:
|
47
50
|
#
|
48
51
|
# queue - the named queue to use, default 'default'
|
49
|
-
# class - the
|
52
|
+
# class - the job class to call, required
|
50
53
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
54
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
55
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
56
|
# backtrace - whether to save any error backtrace, default false
|
54
57
|
#
|
55
58
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
-
#
|
59
|
+
# job options. Otherwise, they will be based on the job class's options.
|
57
60
|
#
|
58
|
-
# Any options valid for a
|
61
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
59
62
|
#
|
60
63
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
61
64
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -64,13 +67,15 @@ module Sidekiq
|
|
64
67
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
65
68
|
#
|
66
69
|
# Example:
|
67
|
-
# push('queue' => 'my_queue', 'class' =>
|
70
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
68
71
|
#
|
69
72
|
def push(item)
|
70
73
|
normed = normalize_item(item)
|
71
|
-
payload =
|
72
|
-
|
74
|
+
payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
|
75
|
+
normed
|
76
|
+
end
|
73
77
|
if payload
|
78
|
+
verify_json(payload)
|
74
79
|
raw_push([payload])
|
75
80
|
payload["jid"]
|
76
81
|
end
|
@@ -98,12 +103,17 @@ module Sidekiq
|
|
98
103
|
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
99
104
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
100
105
|
|
106
|
+
jid = items.delete("jid")
|
107
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
108
|
+
|
101
109
|
normed = normalize_item(items)
|
102
110
|
payloads = args.map.with_index { |job_args, index|
|
103
|
-
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12)
|
111
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
104
112
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
105
|
-
|
106
|
-
|
113
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
114
|
+
verify_json(copy)
|
115
|
+
copy
|
116
|
+
end
|
107
117
|
result || nil
|
108
118
|
}.compact
|
109
119
|
|
@@ -116,8 +126,8 @@ module Sidekiq
|
|
116
126
|
#
|
117
127
|
# pool = ConnectionPool.new { Redis.new }
|
118
128
|
# Sidekiq::Client.via(pool) do
|
119
|
-
#
|
120
|
-
#
|
129
|
+
# SomeJob.perform_async(1,2,3)
|
130
|
+
# SomeOtherJob.perform_async(1,2,3)
|
121
131
|
# end
|
122
132
|
#
|
123
133
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -142,10 +152,10 @@ module Sidekiq
|
|
142
152
|
end
|
143
153
|
|
144
154
|
# Resque compatibility helpers. Note all helpers
|
145
|
-
# should go through
|
155
|
+
# should go through Sidekiq::Job#client_push.
|
146
156
|
#
|
147
157
|
# Example usage:
|
148
|
-
# Sidekiq::Client.enqueue(
|
158
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
149
159
|
#
|
150
160
|
# Messages are enqueued to the 'default' queue.
|
151
161
|
#
|
@@ -154,19 +164,19 @@ module Sidekiq
|
|
154
164
|
end
|
155
165
|
|
156
166
|
# Example usage:
|
157
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
167
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
158
168
|
#
|
159
169
|
def enqueue_to(queue, klass, *args)
|
160
170
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
161
171
|
end
|
162
172
|
|
163
173
|
# Example usage:
|
164
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
174
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
165
175
|
#
|
166
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
167
177
|
int = interval.to_f
|
168
178
|
now = Time.now.to_f
|
169
|
-
ts = (int < 1_000_000_000 ? now + int : int)
|
179
|
+
ts = ((int < 1_000_000_000) ? now + int : int)
|
170
180
|
|
171
181
|
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
172
182
|
item.delete("at") if ts <= now
|
@@ -175,7 +185,7 @@ module Sidekiq
|
|
175
185
|
end
|
176
186
|
|
177
187
|
# Example usage:
|
178
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
188
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
179
189
|
#
|
180
190
|
def enqueue_in(interval, klass, *args)
|
181
191
|
klass.perform_in(interval, *args)
|
@@ -186,8 +196,23 @@ module Sidekiq
|
|
186
196
|
|
187
197
|
def raw_push(payloads)
|
188
198
|
@redis_pool.with do |conn|
|
189
|
-
|
190
|
-
|
199
|
+
retryable = true
|
200
|
+
begin
|
201
|
+
conn.pipelined do |pipeline|
|
202
|
+
atomic_push(pipeline, payloads)
|
203
|
+
end
|
204
|
+
rescue RedisConnection.adapter::BaseError => ex
|
205
|
+
# 2550 Failover can cause the server to become a replica, need
|
206
|
+
# to disconnect and reopen the socket to get back to the primary.
|
207
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
|
+
# The retry logic is copied from sidekiq.rb
|
210
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
+
conn.disconnect!
|
212
|
+
retryable = false
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
raise
|
191
216
|
end
|
192
217
|
end
|
193
218
|
true
|
@@ -195,7 +220,7 @@ module Sidekiq
|
|
195
220
|
|
196
221
|
def atomic_push(conn, payloads)
|
197
222
|
if payloads.first.key?("at")
|
198
|
-
conn.zadd("schedule", payloads.
|
223
|
+
conn.zadd("schedule", payloads.flat_map { |hash|
|
199
224
|
at = hash.delete("at").to_s
|
200
225
|
[at, Sidekiq.dump_json(hash)]
|
201
226
|
})
|
@@ -206,54 +231,9 @@ module Sidekiq
|
|
206
231
|
entry["enqueued_at"] = now
|
207
232
|
Sidekiq.dump_json(entry)
|
208
233
|
}
|
209
|
-
conn.sadd("queues", queue)
|
234
|
+
conn.sadd("queues", [queue])
|
210
235
|
conn.lpush("queue:#{queue}", to_push)
|
211
236
|
end
|
212
237
|
end
|
213
|
-
|
214
|
-
def process_single(worker_class, item)
|
215
|
-
queue = item["queue"]
|
216
|
-
|
217
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
218
|
-
item
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
def validate(item)
|
223
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
224
|
-
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
225
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
226
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
227
|
-
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
228
|
-
end
|
229
|
-
|
230
|
-
def normalize_item(item)
|
231
|
-
validate(item)
|
232
|
-
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
233
|
-
|
234
|
-
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
235
|
-
# this allows ActiveJobs to control sidekiq_options too.
|
236
|
-
defaults = normalized_hash(item["class"])
|
237
|
-
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
238
|
-
item = defaults.merge(item)
|
239
|
-
|
240
|
-
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
241
|
-
|
242
|
-
item["class"] = item["class"].to_s
|
243
|
-
item["queue"] = item["queue"].to_s
|
244
|
-
item["jid"] ||= SecureRandom.hex(12)
|
245
|
-
item["created_at"] ||= Time.now.to_f
|
246
|
-
|
247
|
-
item
|
248
|
-
end
|
249
|
-
|
250
|
-
def normalized_hash(item_class)
|
251
|
-
if item_class.is_a?(Class)
|
252
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
253
|
-
item_class.get_sidekiq_options
|
254
|
-
else
|
255
|
-
Sidekiq.default_worker_options
|
256
|
-
end
|
257
|
-
end
|
258
238
|
end
|
259
239
|
end
|
@@ -1,43 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "forwardable"
|
4
|
-
require "socket"
|
5
|
-
require "securerandom"
|
6
|
-
require "sidekiq/exception_handler"
|
7
|
-
|
8
1
|
module Sidekiq
|
9
2
|
##
|
10
|
-
#
|
11
|
-
#
|
12
|
-
|
13
|
-
class RingBuffer
|
14
|
-
include Enumerable
|
15
|
-
extend Forwardable
|
16
|
-
def_delegators :@buf, :[], :each, :size
|
17
|
-
|
18
|
-
def initialize(size, default = 0)
|
19
|
-
@size = size
|
20
|
-
@buf = Array.new(size, default)
|
21
|
-
@index = 0
|
22
|
-
end
|
23
|
-
|
24
|
-
def <<(element)
|
25
|
-
@buf[@index % @size] = element
|
26
|
-
@index += 1
|
27
|
-
element
|
28
|
-
end
|
29
|
-
|
30
|
-
def buffer
|
31
|
-
@buf
|
32
|
-
end
|
33
|
-
|
34
|
-
def reset(default = 0)
|
35
|
-
@buf.fill(default)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
module Util
|
40
|
-
include ExceptionHandler
|
3
|
+
# Sidekiq::Component assumes a config instance is available at @config
|
4
|
+
module Component # :nodoc:
|
5
|
+
attr_reader :config
|
41
6
|
|
42
7
|
def watchdog(last_words)
|
43
8
|
yield
|
@@ -54,11 +19,11 @@ module Sidekiq
|
|
54
19
|
end
|
55
20
|
|
56
21
|
def logger
|
57
|
-
|
22
|
+
config.logger
|
58
23
|
end
|
59
24
|
|
60
25
|
def redis(&block)
|
61
|
-
|
26
|
+
config.redis(&block)
|
62
27
|
end
|
63
28
|
|
64
29
|
def tid
|
@@ -77,11 +42,16 @@ module Sidekiq
|
|
77
42
|
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
78
43
|
end
|
79
44
|
|
45
|
+
def handle_exception(ex, ctx = {})
|
46
|
+
config.handle_exception(ex, ctx)
|
47
|
+
end
|
48
|
+
|
80
49
|
def fire_event(event, options = {})
|
50
|
+
oneshot = options.fetch(:oneshot, true)
|
81
51
|
reverse = options[:reverse]
|
82
52
|
reraise = options[:reraise]
|
83
53
|
|
84
|
-
arr =
|
54
|
+
arr = config[:lifecycle_events][event]
|
85
55
|
arr.reverse! if reverse
|
86
56
|
arr.each do |block|
|
87
57
|
block.call
|
@@ -89,7 +59,7 @@ module Sidekiq
|
|
89
59
|
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
90
60
|
raise ex if reraise
|
91
61
|
end
|
92
|
-
arr.clear
|
62
|
+
arr.clear if oneshot # once we've fired an event, we never fire it again
|
93
63
|
end
|
94
64
|
end
|
95
65
|
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module Sidekiq
|
3
|
+
module Sidekiq # :nodoc:
|
4
4
|
module Extensions
|
5
5
|
def self.enable_delay!
|
6
|
+
warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
|
7
|
+
|
6
8
|
if defined?(::ActiveSupport)
|
7
9
|
require "sidekiq/extensions/active_record"
|
8
10
|
require "sidekiq/extensions/action_mailer"
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
+
require "sidekiq/component"
|
4
5
|
|
5
|
-
module Sidekiq
|
6
|
+
module Sidekiq # :nodoc:
|
6
7
|
class BasicFetch
|
8
|
+
include Sidekiq::Component
|
7
9
|
# We want the fetch operation to timeout every few seconds so the thread
|
8
10
|
# can check if the process is shutting down.
|
9
11
|
TIMEOUT = 2
|
10
12
|
|
11
|
-
UnitOfWork = Struct.new(:queue, :job) {
|
13
|
+
UnitOfWork = Struct.new(:queue, :job, :config) {
|
12
14
|
def acknowledge
|
13
15
|
# nothing to do
|
14
16
|
end
|
@@ -18,20 +20,20 @@ module Sidekiq
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def requeue
|
21
|
-
|
23
|
+
config.redis do |conn|
|
22
24
|
conn.rpush(queue, job)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
}
|
26
28
|
|
27
|
-
def initialize(
|
28
|
-
raise ArgumentError, "missing queue list" unless
|
29
|
-
@
|
30
|
-
@strictly_ordered_queues = !!@
|
31
|
-
@queues = @
|
29
|
+
def initialize(config)
|
30
|
+
raise ArgumentError, "missing queue list" unless config[:queues]
|
31
|
+
@config = config
|
32
|
+
@strictly_ordered_queues = !!@config[:strict]
|
33
|
+
@queues = @config[:queues].map { |q| "queue:#{q}" }
|
32
34
|
if @strictly_ordered_queues
|
33
35
|
@queues.uniq!
|
34
|
-
@queues << TIMEOUT
|
36
|
+
@queues << {timeout: TIMEOUT}
|
35
37
|
end
|
36
38
|
end
|
37
39
|
|
@@ -44,30 +46,30 @@ module Sidekiq
|
|
44
46
|
return nil
|
45
47
|
end
|
46
48
|
|
47
|
-
|
48
|
-
UnitOfWork.new(
|
49
|
+
queue, job = redis { |conn| conn.brpop(*qs) }
|
50
|
+
UnitOfWork.new(queue, job, config) if queue
|
49
51
|
end
|
50
52
|
|
51
53
|
def bulk_requeue(inprogress, options)
|
52
54
|
return if inprogress.empty?
|
53
55
|
|
54
|
-
|
56
|
+
logger.debug { "Re-queueing terminated jobs" }
|
55
57
|
jobs_to_requeue = {}
|
56
58
|
inprogress.each do |unit_of_work|
|
57
59
|
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
60
|
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
59
61
|
end
|
60
62
|
|
61
|
-
|
62
|
-
conn.pipelined do
|
63
|
+
redis do |conn|
|
64
|
+
conn.pipelined do |pipeline|
|
63
65
|
jobs_to_requeue.each do |queue, jobs|
|
64
|
-
|
66
|
+
pipeline.rpush(queue, jobs)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
68
|
-
|
70
|
+
logger.info("Pushed #{inprogress.size} jobs back to Redis")
|
69
71
|
rescue => ex
|
70
|
-
|
72
|
+
logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
71
73
|
end
|
72
74
|
|
73
75
|
# Creating the Redis#brpop command takes into account any
|
@@ -81,7 +83,7 @@ module Sidekiq
|
|
81
83
|
else
|
82
84
|
permute = @queues.shuffle
|
83
85
|
permute.uniq!
|
84
|
-
permute << TIMEOUT
|
86
|
+
permute << {timeout: TIMEOUT}
|
85
87
|
permute
|
86
88
|
end
|
87
89
|
end
|