sidekiq 5.2.8 → 6.0.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +0 -2
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +70 -0
- data/Changes.md +31 -3
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +12 -0
- data/Gemfile +12 -11
- data/Gemfile.lock +196 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +12 -3
- data/README.md +16 -30
- data/Rakefile +5 -4
- data/bin/sidekiqload +26 -22
- data/bin/sidekiqmon +9 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +12 -14
- data/lib/sidekiq/api.rb +138 -151
- data/lib/sidekiq/cli.rb +97 -162
- data/lib/sidekiq/client.rb +45 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +5 -6
- data/lib/sidekiq/job_logger.rb +37 -7
- data/lib/sidekiq/job_retry.rb +45 -58
- data/lib/sidekiq/launcher.rb +59 -51
- data/lib/sidekiq/logger.rb +69 -0
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +3 -2
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +148 -0
- data/lib/sidekiq/paginator.rb +11 -12
- data/lib/sidekiq/processor.rb +52 -49
- data/lib/sidekiq/rails.rb +23 -29
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +17 -19
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +22 -23
- data/lib/sidekiq/util.rb +17 -14
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +60 -57
- data/lib/sidekiq/web/helpers.rb +66 -67
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/worker.rb +124 -97
- data/lib/sidekiq.rb +53 -42
- data/sidekiq.gemspec +16 -16
- data/web/assets/javascripts/dashboard.js +2 -21
- data/web/locales/ja.yml +2 -1
- metadata +21 -31
- data/.travis.yml +0 -11
- data/bin/sidekiqctl +0 -20
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,29 +1,22 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
$stdout.sync = true
|
3
4
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
9
10
|
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/launcher"
|
13
|
+
require "sidekiq/util"
|
13
14
|
|
14
15
|
module Sidekiq
|
15
16
|
class CLI
|
16
17
|
include Util
|
17
18
|
include Singleton unless $TESTING
|
18
19
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq' },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
20
|
attr_accessor :launcher
|
28
21
|
attr_accessor :environment
|
29
22
|
|
@@ -41,27 +34,19 @@ module Sidekiq
|
|
41
34
|
# global process state irreversibly. PRs which improve the
|
42
35
|
# test coverage of Sidekiq::CLI are welcomed.
|
43
36
|
def run
|
44
|
-
daemonize if options[:daemon]
|
45
|
-
write_pid
|
46
37
|
boot_system
|
47
|
-
|
48
|
-
|
49
|
-
self_read, self_write = IO.pipe
|
50
|
-
sigs = %w(INT TERM TTIN TSTP)
|
51
|
-
# USR1 and USR2 don't work on the JVM
|
52
|
-
if !jruby?
|
53
|
-
sigs << 'USR1'
|
54
|
-
sigs << 'USR2'
|
38
|
+
if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
39
|
+
print_banner
|
55
40
|
end
|
56
41
|
|
42
|
+
self_read, self_write = IO.pipe
|
43
|
+
sigs = %w[INT TERM TTIN TSTP]
|
57
44
|
sigs.each do |sig|
|
58
|
-
|
59
|
-
|
60
|
-
self_write.write("#{sig}\n")
|
61
|
-
end
|
62
|
-
rescue ArgumentError
|
63
|
-
puts "Signal #{sig} not supported"
|
45
|
+
trap sig do
|
46
|
+
self_write.write("#{sig}\n")
|
64
47
|
end
|
48
|
+
rescue ArgumentError
|
49
|
+
puts "Signal #{sig} not supported"
|
65
50
|
end
|
66
51
|
|
67
52
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
@@ -70,9 +55,8 @@ module Sidekiq
|
|
70
55
|
|
71
56
|
# touch the connection pool so it is created before we
|
72
57
|
# fire startup and start multithreading.
|
73
|
-
ver = Sidekiq.redis_info[
|
74
|
-
raise "You are using Redis v#{ver}, Sidekiq requires Redis
|
75
|
-
logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
|
58
|
+
ver = Sidekiq.redis_info["redis_version"]
|
59
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
76
60
|
|
77
61
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
78
62
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
@@ -90,15 +74,15 @@ module Sidekiq
|
|
90
74
|
# Starting here the process will now have multiple threads running.
|
91
75
|
fire_event(:startup, reverse: false, reraise: true)
|
92
76
|
|
93
|
-
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(
|
94
|
-
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(
|
77
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
|
78
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
|
95
79
|
|
96
80
|
launch(self_read)
|
97
81
|
end
|
98
82
|
|
99
83
|
def launch(self_read)
|
100
|
-
if
|
101
|
-
logger.info
|
84
|
+
if environment == "development" && $stdout.tty?
|
85
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
102
86
|
end
|
103
87
|
|
104
88
|
@launcher = Sidekiq::Launcher.new(options)
|
@@ -106,60 +90,70 @@ module Sidekiq
|
|
106
90
|
begin
|
107
91
|
launcher.run
|
108
92
|
|
109
|
-
while readable_io = IO.select([self_read])
|
93
|
+
while (readable_io = IO.select([self_read]))
|
110
94
|
signal = readable_io.first[0].gets.strip
|
111
95
|
handle_signal(signal)
|
112
96
|
end
|
113
97
|
rescue Interrupt
|
114
|
-
logger.info
|
98
|
+
logger.info "Shutting down"
|
115
99
|
launcher.stop
|
116
|
-
# Explicitly exit so busy Processor threads can't block
|
117
|
-
# process shutdown.
|
118
100
|
logger.info "Bye!"
|
101
|
+
|
102
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
103
|
+
#
|
104
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
105
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
106
|
+
# use the TTIN signal to determine where things are stuck.
|
119
107
|
exit(0)
|
120
108
|
end
|
121
109
|
end
|
122
110
|
|
111
|
+
def self.w
|
112
|
+
"\e[37m"
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.r
|
116
|
+
"\e[31m"
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.b
|
120
|
+
"\e[30m"
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.reset
|
124
|
+
"\e[0m"
|
125
|
+
end
|
126
|
+
|
123
127
|
def self.banner
|
124
|
-
%
|
125
|
-
m,
|
126
|
-
`$b
|
127
|
-
.ss, $$: .,d$
|
128
|
-
`$$P,d$P' .,md$P"'
|
129
|
-
,$$$$$
|
130
|
-
.d
|
131
|
-
$$^' `"
|
132
|
-
$: ,$$:
|
133
|
-
`b :$$
|
134
|
-
$$:
|
135
|
-
$$ |____/|_
|
136
|
-
.d$$
|
137
|
-
}
|
128
|
+
%{
|
129
|
+
#{w} m,
|
130
|
+
#{w} `$b
|
131
|
+
#{w} .ss, $$: .,d$
|
132
|
+
#{w} `$$P,d$P' .,md$P"'
|
133
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
134
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
135
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
136
|
+
#{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
137
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
138
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
139
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
140
|
+
#{w} .d$$ #{r} |_|
|
141
|
+
#{reset}}
|
138
142
|
end
|
139
143
|
|
140
144
|
SIGNAL_HANDLERS = {
|
141
145
|
# Ctrl-C in terminal
|
142
|
-
|
146
|
+
"INT" => ->(cli) { raise Interrupt },
|
143
147
|
# TERM is the signal that Sidekiq must exit.
|
144
148
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
145
|
-
|
146
|
-
|
147
|
-
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
148
|
-
cli.launcher.quiet
|
149
|
-
},
|
150
|
-
'TSTP' => ->(cli) {
|
149
|
+
"TERM" => ->(cli) { raise Interrupt },
|
150
|
+
"TSTP" => ->(cli) {
|
151
151
|
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
152
152
|
cli.launcher.quiet
|
153
153
|
},
|
154
|
-
|
155
|
-
if Sidekiq.options[:logfile]
|
156
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
157
|
-
Sidekiq::Logging.reopen_logs
|
158
|
-
end
|
159
|
-
},
|
160
|
-
'TTIN' => ->(cli) {
|
154
|
+
"TTIN" => ->(cli) {
|
161
155
|
Thread.list.each do |thread|
|
162
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread
|
156
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
163
157
|
if thread.backtrace
|
164
158
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
165
159
|
else
|
@@ -182,45 +176,20 @@ module Sidekiq
|
|
182
176
|
private
|
183
177
|
|
184
178
|
def print_banner
|
185
|
-
puts "\e[
|
179
|
+
puts "\e[31m"
|
186
180
|
puts Sidekiq::CLI.banner
|
187
181
|
puts "\e[0m"
|
188
182
|
end
|
189
183
|
|
190
|
-
def daemonize
|
191
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
192
|
-
|
193
|
-
files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
|
194
|
-
::Process.daemon(true, true)
|
195
|
-
|
196
|
-
files_to_reopen.each do |file|
|
197
|
-
begin
|
198
|
-
file.reopen file.path, "a+"
|
199
|
-
file.sync = true
|
200
|
-
rescue ::Exception
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
[$stdout, $stderr].each do |io|
|
205
|
-
File.open(options[:logfile], 'ab') do |f|
|
206
|
-
io.reopen(f)
|
207
|
-
end
|
208
|
-
io.sync = true
|
209
|
-
end
|
210
|
-
$stdin.reopen('/dev/null')
|
211
|
-
|
212
|
-
initialize_logger
|
213
|
-
end
|
214
|
-
|
215
184
|
def set_environment(cli_env)
|
216
|
-
@environment = cli_env || ENV[
|
185
|
+
@environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
217
186
|
end
|
218
187
|
|
219
188
|
def symbolize_keys_deep!(hash)
|
220
189
|
hash.keys.each do |k|
|
221
190
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
222
191
|
hash[symkey] = hash.delete k
|
223
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
192
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
224
193
|
end
|
225
194
|
end
|
226
195
|
|
@@ -240,9 +209,9 @@ module Sidekiq
|
|
240
209
|
end
|
241
210
|
else
|
242
211
|
config_dir = if File.directory?(opts[:require].to_s)
|
243
|
-
File.join(opts[:require],
|
212
|
+
File.join(opts[:require], "config")
|
244
213
|
else
|
245
|
-
File.join(options[:require],
|
214
|
+
File.join(options[:require], "config")
|
246
215
|
end
|
247
216
|
|
248
217
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -255,7 +224,7 @@ module Sidekiq
|
|
255
224
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
256
225
|
|
257
226
|
# set defaults
|
258
|
-
opts[:queues] = Array(opts[:queues]) <<
|
227
|
+
opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
|
259
228
|
opts[:strict] = true if opts[:strict].nil?
|
260
229
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
261
230
|
|
@@ -268,23 +237,14 @@ module Sidekiq
|
|
268
237
|
end
|
269
238
|
|
270
239
|
def boot_system
|
271
|
-
ENV[
|
240
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
272
241
|
|
273
242
|
if File.directory?(options[:require])
|
274
|
-
require
|
275
|
-
if ::Rails::VERSION::MAJOR <
|
243
|
+
require "rails"
|
244
|
+
if ::Rails::VERSION::MAJOR < 5
|
276
245
|
raise "Sidekiq no longer supports this version of Rails"
|
277
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
278
|
-
# Painful contortions, see 1791 for discussion
|
279
|
-
# No autoloading, we want to force eager load for everything.
|
280
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
281
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
282
|
-
::Rails.application.config.eager_load = true
|
283
|
-
end
|
284
|
-
require 'sidekiq/rails'
|
285
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
286
246
|
else
|
287
|
-
require
|
247
|
+
require "sidekiq/rails"
|
288
248
|
require File.expand_path("#{options[:require]}/config/environment.rb")
|
289
249
|
end
|
290
250
|
options[:tag] ||= default_tag
|
@@ -296,8 +256,9 @@ module Sidekiq
|
|
296
256
|
def default_tag
|
297
257
|
dir = ::Rails.root
|
298
258
|
name = File.basename(dir)
|
299
|
-
|
300
|
-
|
259
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
260
|
+
if name.to_i != 0 && prevdir
|
261
|
+
if File.basename(prevdir) == "releases"
|
301
262
|
return File.basename(File.dirname(prevdir))
|
302
263
|
end
|
303
264
|
end
|
@@ -306,9 +267,9 @@ module Sidekiq
|
|
306
267
|
|
307
268
|
def validate!
|
308
269
|
if !File.exist?(options[:require]) ||
|
309
|
-
|
270
|
+
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
310
271
|
logger.info "=================================================================="
|
311
|
-
logger.info " Please point
|
272
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
312
273
|
logger.info " to load your worker classes with -r [DIR|FILE]."
|
313
274
|
logger.info "=================================================================="
|
314
275
|
logger.info @parser
|
@@ -316,47 +277,40 @@ module Sidekiq
|
|
316
277
|
end
|
317
278
|
|
318
279
|
[:concurrency, :timeout].each do |opt|
|
319
|
-
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.
|
280
|
+
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
|
320
281
|
end
|
321
282
|
end
|
322
283
|
|
323
284
|
def parse_options(argv)
|
324
285
|
opts = {}
|
325
286
|
|
326
|
-
@parser = OptionParser.new
|
327
|
-
o.on
|
287
|
+
@parser = OptionParser.new { |o|
|
288
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
328
289
|
opts[:concurrency] = Integer(arg)
|
329
290
|
end
|
330
291
|
|
331
|
-
o.on
|
332
|
-
|
333
|
-
puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
292
|
+
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
293
|
+
puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
334
294
|
end
|
335
295
|
|
336
|
-
o.on
|
296
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
337
297
|
opts[:environment] = arg
|
338
298
|
end
|
339
299
|
|
340
|
-
o.on
|
300
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
341
301
|
opts[:tag] = arg
|
342
302
|
end
|
343
303
|
|
344
|
-
# this index remains here for backwards compatibility but none of the Sidekiq
|
345
|
-
# family use this value anymore. it was used by Pro's original reliable_fetch.
|
346
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
347
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
348
|
-
end
|
349
|
-
|
350
304
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
351
305
|
queue, weight = arg.split(",")
|
352
306
|
parse_queue opts, queue, weight
|
353
307
|
end
|
354
308
|
|
355
|
-
o.on
|
309
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
|
356
310
|
opts[:require] = arg
|
357
311
|
end
|
358
312
|
|
359
|
-
o.on
|
313
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
360
314
|
opts[:timeout] = Integer(arg)
|
361
315
|
end
|
362
316
|
|
@@ -364,25 +318,23 @@ module Sidekiq
|
|
364
318
|
opts[:verbose] = arg
|
365
319
|
end
|
366
320
|
|
367
|
-
o.on
|
321
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
368
322
|
opts[:config_file] = arg
|
369
323
|
end
|
370
324
|
|
371
|
-
o.on
|
372
|
-
|
373
|
-
puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
|
325
|
+
o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
|
326
|
+
puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
374
327
|
end
|
375
328
|
|
376
|
-
o.on
|
377
|
-
|
378
|
-
puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
329
|
+
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
330
|
+
puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
379
331
|
end
|
380
332
|
|
381
|
-
o.on
|
333
|
+
o.on "-V", "--version", "Print version and exit" do |arg|
|
382
334
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
383
335
|
die(0)
|
384
336
|
end
|
385
|
-
|
337
|
+
}
|
386
338
|
|
387
339
|
@parser.banner = "sidekiq [options]"
|
388
340
|
@parser.on_tail "-h", "--help", "Show help" do
|
@@ -396,20 +348,9 @@ module Sidekiq
|
|
396
348
|
end
|
397
349
|
|
398
350
|
def initialize_logger
|
399
|
-
Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
|
400
|
-
|
401
351
|
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
402
352
|
end
|
403
353
|
|
404
|
-
def write_pid
|
405
|
-
if path = options[:pidfile]
|
406
|
-
pidfile = File.expand_path(path)
|
407
|
-
File.open(pidfile, 'w') do |f|
|
408
|
-
f.puts ::Process.pid
|
409
|
-
end
|
410
|
-
end
|
411
|
-
end
|
412
|
-
|
413
354
|
def parse_config(path)
|
414
355
|
opts = YAML.load(ERB.new(File.read(path)).result) || {}
|
415
356
|
|
@@ -422,12 +363,6 @@ module Sidekiq
|
|
422
363
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
423
364
|
parse_queues(opts, opts.delete(:queues) || [])
|
424
365
|
|
425
|
-
ns = opts.delete(:namespace)
|
426
|
-
if ns
|
427
|
-
# logger hasn't been initialized yet, puts is all we have.
|
428
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
429
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
430
|
-
end
|
431
366
|
opts
|
432
367
|
end
|
433
368
|
|
data/lib/sidekiq/client.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/middleware/chain"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Client
|
7
|
-
|
8
8
|
##
|
9
9
|
# Define client-side middleware:
|
10
10
|
#
|
@@ -38,7 +38,7 @@ module Sidekiq
|
|
38
38
|
# Generally this is only needed for very large Sidekiq installs processing
|
39
39
|
# thousands of jobs per second. I don't recommend sharding unless you
|
40
40
|
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
-
def initialize(redis_pool=nil)
|
41
|
+
def initialize(redis_pool = nil)
|
42
42
|
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
43
|
end
|
44
44
|
|
@@ -68,11 +68,11 @@ module Sidekiq
|
|
68
68
|
#
|
69
69
|
def push(item)
|
70
70
|
normed = normalize_item(item)
|
71
|
-
payload = process_single(item[
|
71
|
+
payload = process_single(item["class"], normed)
|
72
72
|
|
73
73
|
if payload
|
74
74
|
raw_push([payload])
|
75
|
-
payload[
|
75
|
+
payload["jid"]
|
76
76
|
end
|
77
77
|
end
|
78
78
|
|
@@ -90,19 +90,19 @@ module Sidekiq
|
|
90
90
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
91
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
92
|
def push_bulk(items)
|
93
|
-
arg = items[
|
93
|
+
arg = items["args"].first
|
94
94
|
return [] unless arg # no jobs to push
|
95
|
-
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]"
|
95
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
|
96
96
|
|
97
97
|
normed = normalize_item(items)
|
98
|
-
payloads = items[
|
99
|
-
copy = normed.merge(
|
100
|
-
result = process_single(items[
|
101
|
-
result
|
102
|
-
|
103
|
-
|
104
|
-
raw_push(payloads)
|
105
|
-
payloads.collect { |payload| payload[
|
98
|
+
payloads = items["args"].map { |args|
|
99
|
+
copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
|
100
|
+
result = process_single(items["class"], copy)
|
101
|
+
result || nil
|
102
|
+
}.compact
|
103
|
+
|
104
|
+
raw_push(payloads) unless payloads.empty?
|
105
|
+
payloads.collect { |payload| payload["jid"] }
|
106
106
|
end
|
107
107
|
|
108
108
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -127,7 +127,6 @@ module Sidekiq
|
|
127
127
|
end
|
128
128
|
|
129
129
|
class << self
|
130
|
-
|
131
130
|
def push(item)
|
132
131
|
new.push(item)
|
133
132
|
end
|
@@ -145,14 +144,14 @@ module Sidekiq
|
|
145
144
|
# Messages are enqueued to the 'default' queue.
|
146
145
|
#
|
147
146
|
def enqueue(klass, *args)
|
148
|
-
klass.client_push(
|
147
|
+
klass.client_push("class" => klass, "args" => args)
|
149
148
|
end
|
150
149
|
|
151
150
|
# Example usage:
|
152
151
|
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
153
152
|
#
|
154
153
|
def enqueue_to(queue, klass, *args)
|
155
|
-
klass.client_push(
|
154
|
+
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
156
155
|
end
|
157
156
|
|
158
157
|
# Example usage:
|
@@ -163,8 +162,8 @@ module Sidekiq
|
|
163
162
|
now = Time.now.to_f
|
164
163
|
ts = (int < 1_000_000_000 ? now + int : int)
|
165
164
|
|
166
|
-
item = {
|
167
|
-
item.delete(
|
165
|
+
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
166
|
+
item.delete("at") if ts <= now
|
168
167
|
|
169
168
|
klass.client_push(item)
|
170
169
|
end
|
@@ -189,25 +188,25 @@ module Sidekiq
|
|
189
188
|
end
|
190
189
|
|
191
190
|
def atomic_push(conn, payloads)
|
192
|
-
if payloads.first[
|
193
|
-
conn.zadd(
|
194
|
-
at = hash.delete(
|
191
|
+
if payloads.first["at"]
|
192
|
+
conn.zadd("schedule", payloads.map { |hash|
|
193
|
+
at = hash.delete("at").to_s
|
195
194
|
[at, Sidekiq.dump_json(hash)]
|
196
|
-
|
195
|
+
})
|
197
196
|
else
|
198
|
-
|
197
|
+
queue = payloads.first["queue"]
|
199
198
|
now = Time.now.to_f
|
200
|
-
to_push = payloads.map
|
201
|
-
entry[
|
199
|
+
to_push = payloads.map { |entry|
|
200
|
+
entry["enqueued_at"] = now
|
202
201
|
Sidekiq.dump_json(entry)
|
203
|
-
|
204
|
-
conn.sadd(
|
205
|
-
conn.lpush("queue:#{
|
202
|
+
}
|
203
|
+
conn.sadd("queues", queue)
|
204
|
+
conn.lpush("queue:#{queue}", to_push)
|
206
205
|
end
|
207
206
|
end
|
208
207
|
|
209
208
|
def process_single(worker_class, item)
|
210
|
-
queue = item[
|
209
|
+
queue = item["queue"]
|
211
210
|
|
212
211
|
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
213
212
|
item
|
@@ -215,25 +214,25 @@ module Sidekiq
|
|
215
214
|
end
|
216
215
|
|
217
216
|
def normalize_item(item)
|
218
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.
|
219
|
-
raise(ArgumentError, "Job args must be an Array") unless item[
|
220
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item[
|
221
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.
|
222
|
-
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
223
|
-
|
224
|
-
normalized_hash(item[
|
225
|
-
.each{ |key, value| item[key] = value if item[key].nil? }
|
226
|
-
|
227
|
-
item[
|
228
|
-
item[
|
229
|
-
item[
|
230
|
-
item[
|
217
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
218
|
+
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
219
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
220
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
221
|
+
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
222
|
+
|
223
|
+
normalized_hash(item["class"])
|
224
|
+
.each { |key, value| item[key] = value if item[key].nil? }
|
225
|
+
|
226
|
+
item["class"] = item["class"].to_s
|
227
|
+
item["queue"] = item["queue"].to_s
|
228
|
+
item["jid"] ||= SecureRandom.hex(12)
|
229
|
+
item["created_at"] ||= Time.now.to_f
|
231
230
|
item
|
232
231
|
end
|
233
232
|
|
234
233
|
def normalized_hash(item_class)
|
235
234
|
if item_class.is_a?(Class)
|
236
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}")
|
235
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
237
236
|
item_class.get_sidekiq_options
|
238
237
|
else
|
239
238
|
Sidekiq.default_worker_options
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Sidekiq
|
3
4
|
module Extensions
|
4
|
-
|
5
5
|
def self.enable_delay!
|
6
6
|
if defined?(::ActiveSupport)
|
7
|
-
require
|
8
|
-
require
|
7
|
+
require "sidekiq/extensions/active_record"
|
8
|
+
require "sidekiq/extensions/action_mailer"
|
9
9
|
|
10
10
|
# Need to patch Psych so it can autoload classes whose names are serialized
|
11
11
|
# in the delayed YAML.
|
@@ -19,7 +19,7 @@ module Sidekiq
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
require
|
22
|
+
require "sidekiq/extensions/class_methods"
|
23
23
|
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
24
24
|
end
|
25
25
|
|
@@ -27,7 +27,7 @@ module Sidekiq
|
|
27
27
|
def resolve_class(klass_name)
|
28
28
|
return nil if !klass_name || klass_name.empty?
|
29
29
|
# constantize
|
30
|
-
names = klass_name.split(
|
30
|
+
names = klass_name.split("::")
|
31
31
|
names.shift if names.empty? || names.first.empty?
|
32
32
|
|
33
33
|
names.inject(Object) do |constant, name|
|
@@ -39,4 +39,3 @@ module Sidekiq
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
end
|
42
|
-
|