sidekiq 6.0.7 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +213 -2
- data/LICENSE +3 -3
- data/README.md +11 -10
- data/bin/sidekiq +8 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +211 -136
- data/lib/sidekiq/cli.rb +81 -46
- data/lib/sidekiq/client.rb +52 -71
- data/lib/sidekiq/{util.rb → component.rb} +11 -14
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +42 -31
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +38 -38
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +123 -63
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +28 -17
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +41 -41
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +87 -53
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +39 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +40 -34
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +136 -16
- data/lib/sidekiq.rb +107 -30
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +113 -65
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +42 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +51 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +30 -51
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,23 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
12
13
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
14
|
|
15
|
-
module Sidekiq
|
15
|
+
module Sidekiq # :nodoc:
|
16
16
|
class CLI
|
17
|
-
include
|
17
|
+
include Sidekiq::Component
|
18
18
|
include Singleton unless $TESTING
|
19
19
|
|
20
20
|
attr_accessor :launcher
|
21
21
|
attr_accessor :environment
|
22
|
+
attr_accessor :config
|
23
|
+
|
24
|
+
def parse(args = ARGV.dup)
|
25
|
+
@config = Sidekiq
|
26
|
+
@config[:error_handlers].clear
|
27
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
22
28
|
|
23
|
-
def parse(args = ARGV)
|
24
29
|
setup_options(args)
|
25
30
|
initialize_logger
|
26
31
|
validate!
|
@@ -33,9 +38,10 @@ module Sidekiq
|
|
33
38
|
# Code within this method is not tested because it alters
|
34
39
|
# global process state irreversibly. PRs which improve the
|
35
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
36
|
-
def run
|
37
|
-
|
38
|
-
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
43
|
+
|
44
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
39
45
|
print_banner
|
40
46
|
end
|
41
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -43,9 +49,17 @@ module Sidekiq
|
|
43
49
|
self_read, self_write = IO.pipe
|
44
50
|
sigs = %w[INT TERM TTIN TSTP]
|
45
51
|
# USR1 and USR2 don't work on the JVM
|
46
|
-
sigs << "USR2"
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
47
53
|
sigs.each do |sig|
|
48
|
-
trap
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
62
|
+
end
|
49
63
|
self_write.puts(sig)
|
50
64
|
end
|
51
65
|
rescue ArgumentError
|
@@ -58,27 +72,40 @@ module Sidekiq
|
|
58
72
|
|
59
73
|
# touch the connection pool so it is created before we
|
60
74
|
# fire startup and start multithreading.
|
61
|
-
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = info["redis_version"]
|
62
77
|
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
63
78
|
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction"
|
81
|
+
logger.warn <<~EOM
|
82
|
+
|
83
|
+
|
84
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
85
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
86
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
87
|
+
|
88
|
+
EOM
|
89
|
+
end
|
90
|
+
|
64
91
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
65
92
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
66
|
-
cursize =
|
67
|
-
needed =
|
93
|
+
cursize = @config.redis_pool.size
|
94
|
+
needed = @config[:concurrency] + 2
|
68
95
|
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
69
96
|
|
70
97
|
# cache process identity
|
71
|
-
|
98
|
+
@config[:identity] = identity
|
72
99
|
|
73
100
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
74
|
-
|
101
|
+
@config.server_middleware
|
75
102
|
|
76
103
|
# Before this point, the process is initializing with just the main thread.
|
77
104
|
# Starting here the process will now have multiple threads running.
|
78
105
|
fire_event(:startup, reverse: false, reraise: true)
|
79
106
|
|
80
|
-
logger.debug { "Client Middleware: #{
|
81
|
-
logger.debug { "Server Middleware: #{
|
107
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
82
109
|
|
83
110
|
launch(self_read)
|
84
111
|
end
|
@@ -88,13 +115,13 @@ module Sidekiq
|
|
88
115
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
89
116
|
end
|
90
117
|
|
91
|
-
@launcher = Sidekiq::Launcher.new(
|
118
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
92
119
|
|
93
120
|
begin
|
94
121
|
launcher.run
|
95
122
|
|
96
|
-
while
|
97
|
-
signal =
|
123
|
+
while self_read.wait_readable
|
124
|
+
signal = self_read.gets.strip
|
98
125
|
handle_signal(signal)
|
99
126
|
end
|
100
127
|
rescue Interrupt
|
@@ -151,25 +178,25 @@ module Sidekiq
|
|
151
178
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
152
179
|
"TERM" => ->(cli) { raise Interrupt },
|
153
180
|
"TSTP" => ->(cli) {
|
154
|
-
|
181
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
155
182
|
cli.launcher.quiet
|
156
183
|
},
|
157
184
|
"TTIN" => ->(cli) {
|
158
185
|
Thread.list.each do |thread|
|
159
|
-
|
186
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
160
187
|
if thread.backtrace
|
161
|
-
|
188
|
+
cli.logger.warn thread.backtrace.join("\n")
|
162
189
|
else
|
163
|
-
|
190
|
+
cli.logger.warn "<no backtrace available>"
|
164
191
|
end
|
165
192
|
end
|
166
193
|
}
|
167
194
|
}
|
168
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
195
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
169
196
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
170
197
|
|
171
198
|
def handle_signal(sig)
|
172
|
-
|
199
|
+
logger.debug "Got #{sig} signal"
|
173
200
|
SIGNAL_HANDLERS[sig].call(self)
|
174
201
|
end
|
175
202
|
|
@@ -215,7 +242,7 @@ module Sidekiq
|
|
215
242
|
config_dir = if File.directory?(opts[:require].to_s)
|
216
243
|
File.join(opts[:require], "config")
|
217
244
|
else
|
218
|
-
File.join(
|
245
|
+
File.join(@config[:require], "config")
|
219
246
|
end
|
220
247
|
|
221
248
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -228,32 +255,27 @@ module Sidekiq
|
|
228
255
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
229
256
|
|
230
257
|
# set defaults
|
231
|
-
opts[:queues] = ["default"] if opts[:queues].nil?
|
232
|
-
opts[:strict] = true if opts[:strict].nil?
|
258
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
233
259
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
234
260
|
|
235
261
|
# merge with defaults
|
236
|
-
|
262
|
+
@config.merge!(opts)
|
237
263
|
end
|
238
264
|
|
239
|
-
def
|
240
|
-
Sidekiq.options
|
241
|
-
end
|
242
|
-
|
243
|
-
def boot_system
|
265
|
+
def boot_application
|
244
266
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
245
267
|
|
246
|
-
if File.directory?(
|
268
|
+
if File.directory?(@config[:require])
|
247
269
|
require "rails"
|
248
270
|
if ::Rails::VERSION::MAJOR < 5
|
249
271
|
raise "Sidekiq no longer supports this version of Rails"
|
250
272
|
else
|
251
273
|
require "sidekiq/rails"
|
252
|
-
require File.expand_path("#{
|
274
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
253
275
|
end
|
254
|
-
|
276
|
+
@config[:tag] ||= default_tag
|
255
277
|
else
|
256
|
-
require
|
278
|
+
require @config[:require]
|
257
279
|
end
|
258
280
|
end
|
259
281
|
|
@@ -270,18 +292,18 @@ module Sidekiq
|
|
270
292
|
end
|
271
293
|
|
272
294
|
def validate!
|
273
|
-
if !File.exist?(
|
274
|
-
(File.directory?(
|
295
|
+
if !File.exist?(@config[:require]) ||
|
296
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
275
297
|
logger.info "=================================================================="
|
276
298
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
277
|
-
logger.info " to load your
|
299
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
278
300
|
logger.info "=================================================================="
|
279
301
|
logger.info @parser
|
280
302
|
die(1)
|
281
303
|
end
|
282
304
|
|
283
305
|
[:concurrency, :timeout].each do |opt|
|
284
|
-
raise ArgumentError, "#{opt}: #{
|
306
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
285
307
|
end
|
286
308
|
end
|
287
309
|
|
@@ -315,7 +337,7 @@ module Sidekiq
|
|
315
337
|
parse_queue opts, queue, weight
|
316
338
|
end
|
317
339
|
|
318
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
340
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
319
341
|
opts[:require] = arg
|
320
342
|
end
|
321
343
|
|
@@ -355,11 +377,13 @@ module Sidekiq
|
|
355
377
|
end
|
356
378
|
|
357
379
|
def initialize_logger
|
358
|
-
|
380
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
359
381
|
end
|
360
382
|
|
361
383
|
def parse_config(path)
|
362
|
-
|
384
|
+
erb = ERB.new(File.read(path))
|
385
|
+
erb.filename = File.expand_path(path)
|
386
|
+
opts = load_yaml(erb.result) || {}
|
363
387
|
|
364
388
|
if opts.respond_to? :deep_symbolize_keys!
|
365
389
|
opts.deep_symbolize_keys!
|
@@ -368,19 +392,30 @@ module Sidekiq
|
|
368
392
|
end
|
369
393
|
|
370
394
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
395
|
+
opts.delete(:strict)
|
396
|
+
|
371
397
|
parse_queues(opts, opts.delete(:queues) || [])
|
372
398
|
|
373
399
|
opts
|
374
400
|
end
|
375
401
|
|
402
|
+
def load_yaml(src)
|
403
|
+
if Psych::VERSION > "4.0"
|
404
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
405
|
+
else
|
406
|
+
YAML.load(src)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
376
410
|
def parse_queues(opts, queues_and_weights)
|
377
411
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
378
412
|
end
|
379
413
|
|
380
414
|
def parse_queue(opts, queue, weight = nil)
|
381
415
|
opts[:queues] ||= []
|
416
|
+
opts[:strict] = true if opts[:strict].nil?
|
382
417
|
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
383
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue }
|
418
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
384
419
|
opts[:strict] = false if weight.to_i > 0
|
385
420
|
end
|
386
421
|
|
data/lib/sidekiq/client.rb
CHANGED
@@ -2,9 +2,12 @@
|
|
2
2
|
|
3
3
|
require "securerandom"
|
4
4
|
require "sidekiq/middleware/chain"
|
5
|
+
require "sidekiq/job_util"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
class Client
|
9
|
+
include Sidekiq::JobUtil
|
10
|
+
|
8
11
|
##
|
9
12
|
# Define client-side middleware:
|
10
13
|
#
|
@@ -12,14 +15,14 @@ module Sidekiq
|
|
12
15
|
# client.middleware do |chain|
|
13
16
|
# chain.use MyClientMiddleware
|
14
17
|
# end
|
15
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
16
19
|
#
|
17
20
|
# All client instances default to the globally-defined
|
18
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
19
22
|
#
|
20
23
|
def middleware(&block)
|
21
24
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
25
|
+
if block
|
23
26
|
@chain = @chain.dup
|
24
27
|
yield @chain
|
25
28
|
end
|
@@ -46,16 +49,16 @@ module Sidekiq
|
|
46
49
|
# The main method used to push a job to Redis. Accepts a number of options:
|
47
50
|
#
|
48
51
|
# queue - the named queue to use, default 'default'
|
49
|
-
# class - the
|
52
|
+
# class - the job class to call, required
|
50
53
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
54
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
55
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
56
|
# backtrace - whether to save any error backtrace, default false
|
54
57
|
#
|
55
58
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
-
#
|
59
|
+
# job options. Otherwise, they will be based on the job class's options.
|
57
60
|
#
|
58
|
-
# Any options valid for a
|
61
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
59
62
|
#
|
60
63
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
61
64
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -64,13 +67,15 @@ module Sidekiq
|
|
64
67
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
65
68
|
#
|
66
69
|
# Example:
|
67
|
-
# push('queue' => 'my_queue', 'class' =>
|
70
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
68
71
|
#
|
69
72
|
def push(item)
|
70
73
|
normed = normalize_item(item)
|
71
|
-
payload =
|
72
|
-
|
74
|
+
payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
|
75
|
+
normed
|
76
|
+
end
|
73
77
|
if payload
|
78
|
+
verify_json(payload)
|
74
79
|
raw_push([payload])
|
75
80
|
payload["jid"]
|
76
81
|
end
|
@@ -90,19 +95,25 @@ module Sidekiq
|
|
90
95
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
96
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
97
|
def push_bulk(items)
|
93
|
-
|
94
|
-
|
95
|
-
|
98
|
+
args = items["args"]
|
99
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
100
|
+
return [] if args.empty? # no jobs to push
|
96
101
|
|
97
102
|
at = items.delete("at")
|
98
|
-
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
|
103
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
104
|
+
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
105
|
+
|
106
|
+
jid = items.delete("jid")
|
107
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
99
108
|
|
100
109
|
normed = normalize_item(items)
|
101
|
-
payloads =
|
102
|
-
copy = normed.merge("args" =>
|
110
|
+
payloads = args.map.with_index { |job_args, index|
|
111
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
103
112
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
104
|
-
|
105
|
-
|
113
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
114
|
+
verify_json(copy)
|
115
|
+
copy
|
116
|
+
end
|
106
117
|
result || nil
|
107
118
|
}.compact
|
108
119
|
|
@@ -115,8 +126,8 @@ module Sidekiq
|
|
115
126
|
#
|
116
127
|
# pool = ConnectionPool.new { Redis.new }
|
117
128
|
# Sidekiq::Client.via(pool) do
|
118
|
-
#
|
119
|
-
#
|
129
|
+
# SomeJob.perform_async(1,2,3)
|
130
|
+
# SomeOtherJob.perform_async(1,2,3)
|
120
131
|
# end
|
121
132
|
#
|
122
133
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -141,10 +152,10 @@ module Sidekiq
|
|
141
152
|
end
|
142
153
|
|
143
154
|
# Resque compatibility helpers. Note all helpers
|
144
|
-
# should go through
|
155
|
+
# should go through Sidekiq::Job#client_push.
|
145
156
|
#
|
146
157
|
# Example usage:
|
147
|
-
# Sidekiq::Client.enqueue(
|
158
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
148
159
|
#
|
149
160
|
# Messages are enqueued to the 'default' queue.
|
150
161
|
#
|
@@ -153,14 +164,14 @@ module Sidekiq
|
|
153
164
|
end
|
154
165
|
|
155
166
|
# Example usage:
|
156
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
167
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
157
168
|
#
|
158
169
|
def enqueue_to(queue, klass, *args)
|
159
170
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
160
171
|
end
|
161
172
|
|
162
173
|
# Example usage:
|
163
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
174
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
164
175
|
#
|
165
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
166
177
|
int = interval.to_f
|
@@ -174,7 +185,7 @@ module Sidekiq
|
|
174
185
|
end
|
175
186
|
|
176
187
|
# Example usage:
|
177
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
188
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
178
189
|
#
|
179
190
|
def enqueue_in(interval, klass, *args)
|
180
191
|
klass.perform_in(interval, *args)
|
@@ -185,8 +196,23 @@ module Sidekiq
|
|
185
196
|
|
186
197
|
def raw_push(payloads)
|
187
198
|
@redis_pool.with do |conn|
|
188
|
-
|
189
|
-
|
199
|
+
retryable = true
|
200
|
+
begin
|
201
|
+
conn.pipelined do |pipeline|
|
202
|
+
atomic_push(pipeline, payloads)
|
203
|
+
end
|
204
|
+
rescue RedisConnection.adapter::BaseError => ex
|
205
|
+
# 2550 Failover can cause the server to become a replica, need
|
206
|
+
# to disconnect and reopen the socket to get back to the primary.
|
207
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
|
+
# The retry logic is copied from sidekiq.rb
|
210
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
+
conn.disconnect!
|
212
|
+
retryable = false
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
raise
|
190
216
|
end
|
191
217
|
end
|
192
218
|
true
|
@@ -194,7 +220,7 @@ module Sidekiq
|
|
194
220
|
|
195
221
|
def atomic_push(conn, payloads)
|
196
222
|
if payloads.first.key?("at")
|
197
|
-
conn.zadd("schedule", payloads.
|
223
|
+
conn.zadd("schedule", payloads.flat_map { |hash|
|
198
224
|
at = hash.delete("at").to_s
|
199
225
|
[at, Sidekiq.dump_json(hash)]
|
200
226
|
})
|
@@ -209,50 +235,5 @@ module Sidekiq
|
|
209
235
|
conn.lpush("queue:#{queue}", to_push)
|
210
236
|
end
|
211
237
|
end
|
212
|
-
|
213
|
-
def process_single(worker_class, item)
|
214
|
-
queue = item["queue"]
|
215
|
-
|
216
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
217
|
-
item
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def normalize_item(item)
|
222
|
-
# 6.0.0 push_bulk bug, #4321
|
223
|
-
# TODO Remove after a while...
|
224
|
-
item.delete("at") if item.key?("at") && item["at"].nil?
|
225
|
-
|
226
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
227
|
-
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
228
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
229
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
230
|
-
raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
|
231
|
-
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
232
|
-
|
233
|
-
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
234
|
-
# this allows ActiveJobs to control sidekiq_options too.
|
235
|
-
defaults = normalized_hash(item["class"])
|
236
|
-
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
237
|
-
item = defaults.merge(item)
|
238
|
-
|
239
|
-
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
240
|
-
|
241
|
-
item["class"] = item["class"].to_s
|
242
|
-
item["queue"] = item["queue"].to_s
|
243
|
-
item["jid"] ||= SecureRandom.hex(12)
|
244
|
-
item["created_at"] ||= Time.now.to_f
|
245
|
-
|
246
|
-
item
|
247
|
-
end
|
248
|
-
|
249
|
-
def normalized_hash(item_class)
|
250
|
-
if item_class.is_a?(Class)
|
251
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
252
|
-
item_class.get_sidekiq_options
|
253
|
-
else
|
254
|
-
Sidekiq.default_worker_options
|
255
|
-
end
|
256
|
-
end
|
257
238
|
end
|
258
239
|
end
|
@@ -1,15 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "socket"
|
4
|
-
require "securerandom"
|
5
|
-
require "sidekiq/exception_handler"
|
6
|
-
|
7
1
|
module Sidekiq
|
8
2
|
##
|
9
|
-
#
|
10
|
-
#
|
11
|
-
|
12
|
-
include ExceptionHandler
|
3
|
+
# Sidekiq::Component assumes a config instance is available at @config
|
4
|
+
module Component # :nodoc:
|
5
|
+
attr_reader :config
|
13
6
|
|
14
7
|
def watchdog(last_words)
|
15
8
|
yield
|
@@ -26,11 +19,11 @@ module Sidekiq
|
|
26
19
|
end
|
27
20
|
|
28
21
|
def logger
|
29
|
-
|
22
|
+
config.logger
|
30
23
|
end
|
31
24
|
|
32
25
|
def redis(&block)
|
33
|
-
|
26
|
+
config.redis(&block)
|
34
27
|
end
|
35
28
|
|
36
29
|
def tid
|
@@ -49,11 +42,15 @@ module Sidekiq
|
|
49
42
|
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
50
43
|
end
|
51
44
|
|
45
|
+
def handle_exception(ex, ctx = {})
|
46
|
+
config.handle_exception(ex, ctx)
|
47
|
+
end
|
48
|
+
|
52
49
|
def fire_event(event, options = {})
|
53
50
|
reverse = options[:reverse]
|
54
51
|
reraise = options[:reraise]
|
55
52
|
|
56
|
-
arr =
|
53
|
+
arr = config[:lifecycle_events][event]
|
57
54
|
arr.reverse! if reverse
|
58
55
|
arr.each do |block|
|
59
56
|
block.call
|
@@ -61,7 +58,7 @@ module Sidekiq
|
|
61
58
|
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
62
59
|
raise ex if reraise
|
63
60
|
end
|
64
|
-
arr.clear
|
61
|
+
arr.clear # once we've fired an event, we never fire it again
|
65
62
|
end
|
66
63
|
end
|
67
64
|
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module Sidekiq
|
3
|
+
module Sidekiq # :nodoc:
|
4
4
|
module Extensions
|
5
5
|
def self.enable_delay!
|
6
|
+
warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
|
7
|
+
|
6
8
|
if defined?(::ActiveSupport)
|
7
9
|
require "sidekiq/extensions/active_record"
|
8
10
|
require "sidekiq/extensions/action_mailer"
|
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
10
10
|
#
|
11
|
+
# @example
|
11
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
12
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
13
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
12
13
|
#
|
13
14
|
# Please note, this is not recommended as this will serialize the entire
|
14
15
|
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# execution to Sidekiq.
|
8
|
+
# Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
|
9
|
+
# execution to Sidekiq.
|
10
10
|
#
|
11
|
-
#
|
12
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.delay.delete_inactive
|
13
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
13
14
|
#
|
14
15
|
class DelayedClass
|
15
16
|
include Sidekiq::Worker
|
@@ -10,7 +10,7 @@ module Sidekiq
|
|
10
10
|
def initialize(performable, target, options = {})
|
11
11
|
@performable = performable
|
12
12
|
@target = target
|
13
|
-
@opts = options
|
13
|
+
@opts = options.transform_keys(&:to_s)
|
14
14
|
end
|
15
15
|
|
16
16
|
def method_missing(name, *args)
|
@@ -24,7 +24,9 @@ module Sidekiq
|
|
24
24
|
if marshalled.size > SIZE_LIMIT
|
25
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
26
26
|
end
|
27
|
-
@performable.client_push({"class" => @performable,
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|