sidekiq 5.2.10 → 6.0.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.standard.yml +20 -0
- data/.travis.yml +5 -2
- data/6.0-Upgrade.md +58 -0
- data/Changes.md +21 -16
- data/Gemfile +15 -10
- data/Rakefile +5 -4
- data/bin/sidekiqctl +1 -10
- data/lib/generators/sidekiq/worker_generator.rb +12 -14
- data/lib/sidekiq/api.rb +133 -148
- data/lib/sidekiq/cli.rb +95 -147
- data/lib/sidekiq/client.rb +44 -45
- data/lib/sidekiq/ctl.rb +35 -109
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +5 -6
- data/lib/sidekiq/job_logger.rb +37 -7
- data/lib/sidekiq/job_retry.rb +45 -58
- data/lib/sidekiq/launcher.rb +59 -48
- data/lib/sidekiq/logger.rb +69 -0
- data/lib/sidekiq/manager.rb +6 -8
- data/lib/sidekiq/middleware/chain.rb +2 -1
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/paginator.rb +11 -12
- data/lib/sidekiq/processor.rb +42 -45
- data/lib/sidekiq/rails.rb +2 -26
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +17 -19
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +22 -23
- data/lib/sidekiq/util.rb +18 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +59 -59
- data/lib/sidekiq/web/helpers.rb +66 -67
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/web.rb +36 -44
- data/lib/sidekiq/worker.rb +12 -13
- data/lib/sidekiq.rb +53 -42
- data/sidekiq.gemspec +7 -7
- metadata +20 -32
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,29 +1,22 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
$stdout.sync = true
|
3
4
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
9
10
|
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/launcher"
|
13
|
+
require "sidekiq/util"
|
13
14
|
|
14
15
|
module Sidekiq
|
15
16
|
class CLI
|
16
17
|
include Util
|
17
18
|
include Singleton unless $TESTING
|
18
19
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq' },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
20
|
attr_accessor :launcher
|
28
21
|
attr_accessor :environment
|
29
22
|
|
@@ -41,27 +34,25 @@ module Sidekiq
|
|
41
34
|
# global process state irreversibly. PRs which improve the
|
42
35
|
# test coverage of Sidekiq::CLI are welcomed.
|
43
36
|
def run
|
44
|
-
daemonize if options[:daemon]
|
45
|
-
write_pid
|
46
37
|
boot_system
|
47
|
-
|
38
|
+
if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
39
|
+
print_banner
|
40
|
+
end
|
48
41
|
|
49
42
|
self_read, self_write = IO.pipe
|
50
|
-
sigs = %w
|
43
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
44
|
# USR1 and USR2 don't work on the JVM
|
52
|
-
|
53
|
-
sigs <<
|
54
|
-
sigs <<
|
45
|
+
unless jruby?
|
46
|
+
sigs << "USR1"
|
47
|
+
sigs << "USR2"
|
55
48
|
end
|
56
49
|
|
57
50
|
sigs.each do |sig|
|
58
|
-
|
59
|
-
|
60
|
-
self_write.write("#{sig}\n")
|
61
|
-
end
|
62
|
-
rescue ArgumentError
|
63
|
-
puts "Signal #{sig} not supported"
|
51
|
+
trap sig do
|
52
|
+
self_write.write("#{sig}\n")
|
64
53
|
end
|
54
|
+
rescue ArgumentError
|
55
|
+
puts "Signal #{sig} not supported"
|
65
56
|
end
|
66
57
|
|
67
58
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
@@ -70,9 +61,9 @@ module Sidekiq
|
|
70
61
|
|
71
62
|
# touch the connection pool so it is created before we
|
72
63
|
# fire startup and start multithreading.
|
73
|
-
ver = Sidekiq.redis_info[
|
74
|
-
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver <
|
75
|
-
logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver <
|
64
|
+
ver = Sidekiq.redis_info["redis_version"]
|
65
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < "2.8"
|
66
|
+
logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < "4"
|
76
67
|
|
77
68
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
78
69
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
@@ -90,15 +81,15 @@ module Sidekiq
|
|
90
81
|
# Starting here the process will now have multiple threads running.
|
91
82
|
fire_event(:startup, reverse: false, reraise: true)
|
92
83
|
|
93
|
-
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(
|
94
|
-
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(
|
84
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
|
85
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
|
95
86
|
|
96
87
|
launch(self_read)
|
97
88
|
end
|
98
89
|
|
99
90
|
def launch(self_read)
|
100
|
-
if
|
101
|
-
logger.info
|
91
|
+
if environment == "development" && $stdout.tty?
|
92
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
102
93
|
end
|
103
94
|
|
104
95
|
@launcher = Sidekiq::Launcher.new(options)
|
@@ -106,12 +97,12 @@ module Sidekiq
|
|
106
97
|
begin
|
107
98
|
launcher.run
|
108
99
|
|
109
|
-
while readable_io = IO.select([self_read])
|
100
|
+
while (readable_io = IO.select([self_read]))
|
110
101
|
signal = readable_io.first[0].gets.strip
|
111
102
|
handle_signal(signal)
|
112
103
|
end
|
113
104
|
rescue Interrupt
|
114
|
-
logger.info
|
105
|
+
logger.info "Shutting down"
|
115
106
|
launcher.stop
|
116
107
|
# Explicitly exit so busy Processor threads can't block
|
117
108
|
# process shutdown.
|
@@ -120,46 +111,56 @@ module Sidekiq
|
|
120
111
|
end
|
121
112
|
end
|
122
113
|
|
114
|
+
def self.w
|
115
|
+
"\e[37m"
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.r
|
119
|
+
"\e[31m"
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.b
|
123
|
+
"\e[30m"
|
124
|
+
end
|
125
|
+
|
126
|
+
def self.reset
|
127
|
+
"\e[0m"
|
128
|
+
end
|
129
|
+
|
123
130
|
def self.banner
|
124
|
-
%
|
125
|
-
m,
|
126
|
-
`$b
|
127
|
-
.ss, $$: .,d$
|
128
|
-
`$$P,d$P' .,md$P"'
|
129
|
-
,$$$$$
|
130
|
-
.d
|
131
|
-
$$^' `"
|
132
|
-
$: ,$$:
|
133
|
-
`b :$$
|
134
|
-
$$:
|
135
|
-
$$ |____/|_
|
136
|
-
.d$$
|
137
|
-
}
|
131
|
+
%{
|
132
|
+
#{w} m,
|
133
|
+
#{w} `$b
|
134
|
+
#{w} .ss, $$: .,d$
|
135
|
+
#{w} `$$P,d$P' .,md$P"'
|
136
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
137
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
138
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
139
|
+
#{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
140
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
141
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
142
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
143
|
+
#{w} .d$$ #{r} |_|
|
144
|
+
#{reset}}
|
138
145
|
end
|
139
146
|
|
140
147
|
SIGNAL_HANDLERS = {
|
141
148
|
# Ctrl-C in terminal
|
142
|
-
|
149
|
+
"INT" => ->(cli) { raise Interrupt },
|
143
150
|
# TERM is the signal that Sidekiq must exit.
|
144
151
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
145
|
-
|
146
|
-
|
152
|
+
"TERM" => ->(cli) { raise Interrupt },
|
153
|
+
"USR1" => ->(cli) {
|
147
154
|
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
148
155
|
cli.launcher.quiet
|
149
156
|
},
|
150
|
-
|
157
|
+
"TSTP" => ->(cli) {
|
151
158
|
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
152
159
|
cli.launcher.quiet
|
153
160
|
},
|
154
|
-
|
155
|
-
if Sidekiq.options[:logfile]
|
156
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
157
|
-
Sidekiq::Logging.reopen_logs
|
158
|
-
end
|
159
|
-
},
|
160
|
-
'TTIN' => ->(cli) {
|
161
|
+
"TTIN" => ->(cli) {
|
161
162
|
Thread.list.each do |thread|
|
162
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread[
|
163
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["sidekiq_label"]}"
|
163
164
|
if thread.backtrace
|
164
165
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
165
166
|
else
|
@@ -182,45 +183,20 @@ module Sidekiq
|
|
182
183
|
private
|
183
184
|
|
184
185
|
def print_banner
|
185
|
-
puts "\e[
|
186
|
+
puts "\e[31m"
|
186
187
|
puts Sidekiq::CLI.banner
|
187
188
|
puts "\e[0m"
|
188
189
|
end
|
189
190
|
|
190
|
-
def daemonize
|
191
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
192
|
-
|
193
|
-
files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
|
194
|
-
::Process.daemon(true, true)
|
195
|
-
|
196
|
-
files_to_reopen.each do |file|
|
197
|
-
begin
|
198
|
-
file.reopen file.path, "a+"
|
199
|
-
file.sync = true
|
200
|
-
rescue ::Exception
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
[$stdout, $stderr].each do |io|
|
205
|
-
File.open(options[:logfile], 'ab') do |f|
|
206
|
-
io.reopen(f)
|
207
|
-
end
|
208
|
-
io.sync = true
|
209
|
-
end
|
210
|
-
$stdin.reopen('/dev/null')
|
211
|
-
|
212
|
-
initialize_logger
|
213
|
-
end
|
214
|
-
|
215
191
|
def set_environment(cli_env)
|
216
|
-
@environment = cli_env || ENV[
|
192
|
+
@environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
217
193
|
end
|
218
194
|
|
219
195
|
def symbolize_keys_deep!(hash)
|
220
196
|
hash.keys.each do |k|
|
221
197
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
222
198
|
hash[symkey] = hash.delete k
|
223
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
199
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
224
200
|
end
|
225
201
|
end
|
226
202
|
|
@@ -240,9 +216,9 @@ module Sidekiq
|
|
240
216
|
end
|
241
217
|
else
|
242
218
|
config_dir = if File.directory?(opts[:require].to_s)
|
243
|
-
File.join(opts[:require],
|
219
|
+
File.join(opts[:require], "config")
|
244
220
|
else
|
245
|
-
File.join(options[:require],
|
221
|
+
File.join(options[:require], "config")
|
246
222
|
end
|
247
223
|
|
248
224
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -255,7 +231,7 @@ module Sidekiq
|
|
255
231
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
256
232
|
|
257
233
|
# set defaults
|
258
|
-
opts[:queues] = Array(opts[:queues]) <<
|
234
|
+
opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
|
259
235
|
opts[:strict] = true if opts[:strict].nil?
|
260
236
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
261
237
|
|
@@ -268,23 +244,14 @@ module Sidekiq
|
|
268
244
|
end
|
269
245
|
|
270
246
|
def boot_system
|
271
|
-
ENV[
|
247
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
272
248
|
|
273
249
|
if File.directory?(options[:require])
|
274
|
-
require
|
275
|
-
if ::Rails::VERSION::MAJOR <
|
250
|
+
require "rails"
|
251
|
+
if ::Rails::VERSION::MAJOR < 5
|
276
252
|
raise "Sidekiq no longer supports this version of Rails"
|
277
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
278
|
-
# Painful contortions, see 1791 for discussion
|
279
|
-
# No autoloading, we want to force eager load for everything.
|
280
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
281
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
282
|
-
::Rails.application.config.eager_load = true
|
283
|
-
end
|
284
|
-
require 'sidekiq/rails'
|
285
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
286
253
|
else
|
287
|
-
require
|
254
|
+
require "sidekiq/rails"
|
288
255
|
require File.expand_path("#{options[:require]}/config/environment.rb")
|
289
256
|
end
|
290
257
|
options[:tag] ||= default_tag
|
@@ -296,8 +263,9 @@ module Sidekiq
|
|
296
263
|
def default_tag
|
297
264
|
dir = ::Rails.root
|
298
265
|
name = File.basename(dir)
|
299
|
-
|
300
|
-
|
266
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
267
|
+
if name.to_i != 0 && prevdir
|
268
|
+
if File.basename(prevdir) == "releases"
|
301
269
|
return File.basename(File.dirname(prevdir))
|
302
270
|
end
|
303
271
|
end
|
@@ -306,7 +274,7 @@ module Sidekiq
|
|
306
274
|
|
307
275
|
def validate!
|
308
276
|
if !File.exist?(options[:require]) ||
|
309
|
-
|
277
|
+
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
310
278
|
logger.info "=================================================================="
|
311
279
|
logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
|
312
280
|
logger.info " to load your worker classes with -r [DIR|FILE]."
|
@@ -316,47 +284,40 @@ module Sidekiq
|
|
316
284
|
end
|
317
285
|
|
318
286
|
[:concurrency, :timeout].each do |opt|
|
319
|
-
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.
|
287
|
+
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
|
320
288
|
end
|
321
289
|
end
|
322
290
|
|
323
291
|
def parse_options(argv)
|
324
292
|
opts = {}
|
325
293
|
|
326
|
-
@parser = OptionParser.new
|
327
|
-
o.on
|
294
|
+
@parser = OptionParser.new { |o|
|
295
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
328
296
|
opts[:concurrency] = Integer(arg)
|
329
297
|
end
|
330
298
|
|
331
|
-
o.on
|
332
|
-
|
333
|
-
puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
299
|
+
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
300
|
+
puts "WARNING: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
334
301
|
end
|
335
302
|
|
336
|
-
o.on
|
303
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
337
304
|
opts[:environment] = arg
|
338
305
|
end
|
339
306
|
|
340
|
-
o.on
|
307
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
341
308
|
opts[:tag] = arg
|
342
309
|
end
|
343
310
|
|
344
|
-
# this index remains here for backwards compatibility but none of the Sidekiq
|
345
|
-
# family use this value anymore. it was used by Pro's original reliable_fetch.
|
346
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
347
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
348
|
-
end
|
349
|
-
|
350
311
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
351
312
|
queue, weight = arg.split(",")
|
352
313
|
parse_queue opts, queue, weight
|
353
314
|
end
|
354
315
|
|
355
|
-
o.on
|
316
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
|
356
317
|
opts[:require] = arg
|
357
318
|
end
|
358
319
|
|
359
|
-
o.on
|
320
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
360
321
|
opts[:timeout] = Integer(arg)
|
361
322
|
end
|
362
323
|
|
@@ -364,25 +325,23 @@ module Sidekiq
|
|
364
325
|
opts[:verbose] = arg
|
365
326
|
end
|
366
327
|
|
367
|
-
o.on
|
328
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
368
329
|
opts[:config_file] = arg
|
369
330
|
end
|
370
331
|
|
371
|
-
o.on
|
372
|
-
|
373
|
-
puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
|
332
|
+
o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
|
333
|
+
puts "WARNING: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
374
334
|
end
|
375
335
|
|
376
|
-
o.on
|
377
|
-
|
378
|
-
puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
336
|
+
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
337
|
+
puts "WARNING: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
379
338
|
end
|
380
339
|
|
381
|
-
o.on
|
340
|
+
o.on "-V", "--version", "Print version and exit" do |arg|
|
382
341
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
383
342
|
die(0)
|
384
343
|
end
|
385
|
-
|
344
|
+
}
|
386
345
|
|
387
346
|
@parser.banner = "sidekiq [options]"
|
388
347
|
@parser.on_tail "-h", "--help", "Show help" do
|
@@ -396,20 +355,9 @@ module Sidekiq
|
|
396
355
|
end
|
397
356
|
|
398
357
|
def initialize_logger
|
399
|
-
Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
|
400
|
-
|
401
358
|
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
402
359
|
end
|
403
360
|
|
404
|
-
def write_pid
|
405
|
-
if path = options[:pidfile]
|
406
|
-
pidfile = File.expand_path(path)
|
407
|
-
File.open(pidfile, 'w') do |f|
|
408
|
-
f.puts ::Process.pid
|
409
|
-
end
|
410
|
-
end
|
411
|
-
end
|
412
|
-
|
413
361
|
def parse_config(path)
|
414
362
|
opts = YAML.load(ERB.new(File.read(path)).result) || {}
|
415
363
|
|
data/lib/sidekiq/client.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/middleware/chain"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Client
|
7
|
-
|
8
8
|
##
|
9
9
|
# Define client-side middleware:
|
10
10
|
#
|
@@ -38,7 +38,7 @@ module Sidekiq
|
|
38
38
|
# Generally this is only needed for very large Sidekiq installs processing
|
39
39
|
# thousands of jobs per second. I don't recommend sharding unless you
|
40
40
|
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
-
def initialize(redis_pool=nil)
|
41
|
+
def initialize(redis_pool = nil)
|
42
42
|
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
43
|
end
|
44
44
|
|
@@ -68,11 +68,11 @@ module Sidekiq
|
|
68
68
|
#
|
69
69
|
def push(item)
|
70
70
|
normed = normalize_item(item)
|
71
|
-
payload = process_single(item[
|
71
|
+
payload = process_single(item["class"], normed)
|
72
72
|
|
73
73
|
if payload
|
74
74
|
raw_push([payload])
|
75
|
-
payload[
|
75
|
+
payload["jid"]
|
76
76
|
end
|
77
77
|
end
|
78
78
|
|
@@ -90,19 +90,19 @@ module Sidekiq
|
|
90
90
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
91
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
92
|
def push_bulk(items)
|
93
|
-
arg = items[
|
93
|
+
arg = items["args"].first
|
94
94
|
return [] unless arg # no jobs to push
|
95
|
-
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]"
|
95
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
|
96
96
|
|
97
97
|
normed = normalize_item(items)
|
98
|
-
payloads = items[
|
99
|
-
copy = normed.merge(
|
100
|
-
result = process_single(items[
|
101
|
-
result
|
102
|
-
|
103
|
-
|
104
|
-
raw_push(payloads)
|
105
|
-
payloads.collect { |payload| payload[
|
98
|
+
payloads = items["args"].map { |args|
|
99
|
+
copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
|
100
|
+
result = process_single(items["class"], copy)
|
101
|
+
result || nil
|
102
|
+
}.compact
|
103
|
+
|
104
|
+
raw_push(payloads) unless payloads.empty?
|
105
|
+
payloads.collect { |payload| payload["jid"] }
|
106
106
|
end
|
107
107
|
|
108
108
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -127,7 +127,6 @@ module Sidekiq
|
|
127
127
|
end
|
128
128
|
|
129
129
|
class << self
|
130
|
-
|
131
130
|
def push(item)
|
132
131
|
new.push(item)
|
133
132
|
end
|
@@ -145,14 +144,14 @@ module Sidekiq
|
|
145
144
|
# Messages are enqueued to the 'default' queue.
|
146
145
|
#
|
147
146
|
def enqueue(klass, *args)
|
148
|
-
klass.client_push(
|
147
|
+
klass.client_push("class" => klass, "args" => args)
|
149
148
|
end
|
150
149
|
|
151
150
|
# Example usage:
|
152
151
|
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
153
152
|
#
|
154
153
|
def enqueue_to(queue, klass, *args)
|
155
|
-
klass.client_push(
|
154
|
+
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
156
155
|
end
|
157
156
|
|
158
157
|
# Example usage:
|
@@ -163,8 +162,8 @@ module Sidekiq
|
|
163
162
|
now = Time.now.to_f
|
164
163
|
ts = (int < 1_000_000_000 ? now + int : int)
|
165
164
|
|
166
|
-
item = {
|
167
|
-
item.delete(
|
165
|
+
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
166
|
+
item.delete("at") if ts <= now
|
168
167
|
|
169
168
|
klass.client_push(item)
|
170
169
|
end
|
@@ -189,25 +188,25 @@ module Sidekiq
|
|
189
188
|
end
|
190
189
|
|
191
190
|
def atomic_push(conn, payloads)
|
192
|
-
if payloads.first[
|
193
|
-
conn.zadd(
|
194
|
-
at = hash.delete(
|
191
|
+
if payloads.first["at"]
|
192
|
+
conn.zadd("schedule", payloads.map { |hash|
|
193
|
+
at = hash.delete("at").to_s
|
195
194
|
[at, Sidekiq.dump_json(hash)]
|
196
|
-
|
195
|
+
})
|
197
196
|
else
|
198
|
-
q = payloads.first[
|
197
|
+
q = payloads.first["queue"]
|
199
198
|
now = Time.now.to_f
|
200
|
-
to_push = payloads.map
|
201
|
-
entry[
|
199
|
+
to_push = payloads.map { |entry|
|
200
|
+
entry["enqueued_at"] = now
|
202
201
|
Sidekiq.dump_json(entry)
|
203
|
-
|
204
|
-
conn.sadd(
|
202
|
+
}
|
203
|
+
conn.sadd("queues", q)
|
205
204
|
conn.lpush("queue:#{q}", to_push)
|
206
205
|
end
|
207
206
|
end
|
208
207
|
|
209
208
|
def process_single(worker_class, item)
|
210
|
-
queue = item[
|
209
|
+
queue = item["queue"]
|
211
210
|
|
212
211
|
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
213
212
|
item
|
@@ -215,25 +214,25 @@ module Sidekiq
|
|
215
214
|
end
|
216
215
|
|
217
216
|
def normalize_item(item)
|
218
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.
|
219
|
-
raise(ArgumentError, "Job args must be an Array") unless item[
|
220
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item[
|
221
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.
|
222
|
-
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
223
|
-
|
224
|
-
normalized_hash(item[
|
225
|
-
.each{ |key, value| item[key] = value if item[key].nil? }
|
226
|
-
|
227
|
-
item[
|
228
|
-
item[
|
229
|
-
item[
|
230
|
-
item[
|
217
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
218
|
+
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
219
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
220
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
221
|
+
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
222
|
+
|
223
|
+
normalized_hash(item["class"])
|
224
|
+
.each { |key, value| item[key] = value if item[key].nil? }
|
225
|
+
|
226
|
+
item["class"] = item["class"].to_s
|
227
|
+
item["queue"] = item["queue"].to_s
|
228
|
+
item["jid"] ||= SecureRandom.hex(12)
|
229
|
+
item["created_at"] ||= Time.now.to_f
|
231
230
|
item
|
232
231
|
end
|
233
232
|
|
234
233
|
def normalized_hash(item_class)
|
235
234
|
if item_class.is_a?(Class)
|
236
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}")
|
235
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
237
236
|
item_class.get_sidekiq_options
|
238
237
|
else
|
239
238
|
Sidekiq.default_worker_options
|