sidekiq 5.0.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/contributing.md +32 -0
- data/.github/issue_template.md +9 -0
- data/.gitignore +13 -0
- data/.travis.yml +18 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +53 -0
- data/5.0-Upgrade.md +56 -0
- data/COMM-LICENSE +95 -0
- data/Changes.md +1402 -0
- data/Ent-Changes.md +174 -0
- data/Gemfile +29 -0
- data/LICENSE +9 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-Changes.md +632 -0
- data/README.md +107 -0
- data/Rakefile +12 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +99 -0
- data/bin/sidekiqload +149 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq.rb +228 -0
- data/lib/sidekiq/api.rb +871 -0
- data/lib/sidekiq/cli.rb +413 -0
- data/lib/sidekiq/client.rb +238 -0
- data/lib/sidekiq/core_ext.rb +119 -0
- data/lib/sidekiq/delay.rb +21 -0
- data/lib/sidekiq/exception_handler.rb +31 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/job_logger.rb +27 -0
- data/lib/sidekiq/job_retry.rb +235 -0
- data/lib/sidekiq/launcher.rb +167 -0
- data/lib/sidekiq/logging.rb +106 -0
- data/lib/sidekiq/manager.rb +138 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +22 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +238 -0
- data/lib/sidekiq/rails.rb +60 -0
- data/lib/sidekiq/redis_connection.rb +106 -0
- data/lib/sidekiq/scheduled.rb +147 -0
- data/lib/sidekiq/testing.rb +324 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/util.rb +63 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +213 -0
- data/lib/sidekiq/web/action.rb +89 -0
- data/lib/sidekiq/web/application.rb +331 -0
- data/lib/sidekiq/web/helpers.rb +286 -0
- data/lib/sidekiq/web/router.rb +100 -0
- data/lib/sidekiq/worker.rb +144 -0
- data/sidekiq.gemspec +32 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status.png +0 -0
- data/web/assets/javascripts/application.js +92 -0
- data/web/assets/javascripts/dashboard.js +298 -0
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +1111 -0
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +5 -0
- data/web/locales/ar.yml +80 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +79 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +78 -0
- data/web/locales/he.yml +79 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +78 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +78 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +76 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +17 -0
- data/web/views/_job_info.erb +88 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +94 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +40 -0
- data/web/views/morgue.erb +75 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +28 -0
- data/web/views/retries.erb +76 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +366 -0
data/lib/sidekiq/cli.rb
ADDED
@@ -0,0 +1,413 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
$stdout.sync = true
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'singleton'
|
7
|
+
require 'optparse'
|
8
|
+
require 'erb'
|
9
|
+
require 'fileutils'
|
10
|
+
|
11
|
+
require 'sidekiq'
|
12
|
+
require 'sidekiq/util'
|
13
|
+
|
14
|
+
module Sidekiq
|
15
|
+
class CLI
|
16
|
+
include Util
|
17
|
+
include Singleton unless $TESTING
|
18
|
+
|
19
|
+
PROCTITLES = [
|
20
|
+
proc { 'sidekiq'.freeze },
|
21
|
+
proc { Sidekiq::VERSION },
|
22
|
+
proc { |me, data| data['tag'] },
|
23
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
+
proc { |me, data| "stopping" if me.stopping? },
|
25
|
+
]
|
26
|
+
|
27
|
+
# Used for CLI testing
|
28
|
+
attr_accessor :code
|
29
|
+
attr_accessor :launcher
|
30
|
+
attr_accessor :environment
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
@code = nil
|
34
|
+
end
|
35
|
+
|
36
|
+
def parse(args=ARGV)
|
37
|
+
@code = nil
|
38
|
+
|
39
|
+
setup_options(args)
|
40
|
+
initialize_logger
|
41
|
+
validate!
|
42
|
+
daemonize
|
43
|
+
write_pid
|
44
|
+
end
|
45
|
+
|
46
|
+
def jruby?
|
47
|
+
defined?(::JRUBY_VERSION)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Code within this method is not tested because it alters
|
51
|
+
# global process state irreversibly. PRs which improve the
|
52
|
+
# test coverage of Sidekiq::CLI are welcomed.
|
53
|
+
def run
|
54
|
+
boot_system
|
55
|
+
print_banner
|
56
|
+
|
57
|
+
self_read, self_write = IO.pipe
|
58
|
+
sigs = %w(INT TERM TTIN TSTP)
|
59
|
+
# USR1 and USR2 don't work on the JVM
|
60
|
+
if !jruby?
|
61
|
+
sigs << 'USR1'
|
62
|
+
sigs << 'USR2'
|
63
|
+
end
|
64
|
+
|
65
|
+
sigs.each do |sig|
|
66
|
+
begin
|
67
|
+
trap sig do
|
68
|
+
self_write.puts(sig)
|
69
|
+
end
|
70
|
+
rescue ArgumentError
|
71
|
+
puts "Signal #{sig} not supported"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
76
|
+
logger.info Sidekiq::LICENSE
|
77
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
78
|
+
|
79
|
+
# touch the connection pool so it is created before we
|
80
|
+
# fire startup and start multithreading.
|
81
|
+
ver = Sidekiq.redis_info['redis_version']
|
82
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
|
83
|
+
|
84
|
+
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
85
|
+
Sidekiq.server_middleware
|
86
|
+
|
87
|
+
# Before this point, the process is initializing with just the main thread.
|
88
|
+
# Starting here the process will now have multiple threads running.
|
89
|
+
fire_event(:startup)
|
90
|
+
|
91
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
|
92
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
|
93
|
+
|
94
|
+
if !options[:daemon]
|
95
|
+
logger.info 'Starting processing, hit Ctrl-C to stop'
|
96
|
+
end
|
97
|
+
|
98
|
+
require 'sidekiq/launcher'
|
99
|
+
@launcher = Sidekiq::Launcher.new(options)
|
100
|
+
|
101
|
+
begin
|
102
|
+
launcher.run
|
103
|
+
|
104
|
+
while readable_io = IO.select([self_read])
|
105
|
+
signal = readable_io.first[0].gets.strip
|
106
|
+
handle_signal(signal)
|
107
|
+
end
|
108
|
+
rescue Interrupt
|
109
|
+
logger.info 'Shutting down'
|
110
|
+
launcher.stop
|
111
|
+
# Explicitly exit so busy Processor threads can't block
|
112
|
+
# process shutdown.
|
113
|
+
logger.info "Bye!"
|
114
|
+
exit(0)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.banner
|
119
|
+
%q{
|
120
|
+
m,
|
121
|
+
`$b
|
122
|
+
.ss, $$: .,d$
|
123
|
+
`$$P,d$P' .,md$P"'
|
124
|
+
,$$$$$bmmd$$$P^'
|
125
|
+
.d$$$$$$$$$$P'
|
126
|
+
$$^' `"^$$$' ____ _ _ _ _
|
127
|
+
$: ,$$: / ___|(_) __| | ___| | _(_) __ _
|
128
|
+
`b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
|
129
|
+
$$: ___) | | (_| | __/ <| | (_| |
|
130
|
+
$$ |____/|_|\__,_|\___|_|\_\_|\__, |
|
131
|
+
.d$$ |_|
|
132
|
+
}
|
133
|
+
end
|
134
|
+
|
135
|
+
def handle_signal(sig)
|
136
|
+
Sidekiq.logger.debug "Got #{sig} signal"
|
137
|
+
case sig
|
138
|
+
when 'INT'
|
139
|
+
# Handle Ctrl-C in JRuby like MRI
|
140
|
+
# http://jira.codehaus.org/browse/JRUBY-4637
|
141
|
+
raise Interrupt
|
142
|
+
when 'TERM'
|
143
|
+
# Heroku sends TERM and then waits 10 seconds for process to exit.
|
144
|
+
raise Interrupt
|
145
|
+
when 'USR1'
|
146
|
+
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
147
|
+
launcher.quiet
|
148
|
+
when 'TSTP'
|
149
|
+
# USR1 is not available on JVM, allow TSTP as an alternate signal
|
150
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
151
|
+
launcher.quiet
|
152
|
+
when 'USR2'
|
153
|
+
if Sidekiq.options[:logfile]
|
154
|
+
Sidekiq.logger.info "Received USR2, reopening log file"
|
155
|
+
Sidekiq::Logging.reopen_logs
|
156
|
+
end
|
157
|
+
when 'TTIN'
|
158
|
+
Thread.list.each do |thread|
|
159
|
+
Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['sidekiq_label']}"
|
160
|
+
if thread.backtrace
|
161
|
+
Sidekiq.logger.warn thread.backtrace.join("\n")
|
162
|
+
else
|
163
|
+
Sidekiq.logger.warn "<no backtrace available>"
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def print_banner
|
172
|
+
# Print logo and banner for development
|
173
|
+
if environment == 'development' && $stdout.tty?
|
174
|
+
puts "\e[#{31}m"
|
175
|
+
puts Sidekiq::CLI.banner
|
176
|
+
puts "\e[0m"
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
def daemonize
|
181
|
+
return unless options[:daemon]
|
182
|
+
|
183
|
+
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
184
|
+
files_to_reopen = []
|
185
|
+
ObjectSpace.each_object(File) do |file|
|
186
|
+
files_to_reopen << file unless file.closed?
|
187
|
+
end
|
188
|
+
|
189
|
+
::Process.daemon(true, true)
|
190
|
+
|
191
|
+
files_to_reopen.each do |file|
|
192
|
+
begin
|
193
|
+
file.reopen file.path, "a+"
|
194
|
+
file.sync = true
|
195
|
+
rescue ::Exception
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
[$stdout, $stderr].each do |io|
|
200
|
+
File.open(options[:logfile], 'ab') do |f|
|
201
|
+
io.reopen(f)
|
202
|
+
end
|
203
|
+
io.sync = true
|
204
|
+
end
|
205
|
+
$stdin.reopen('/dev/null')
|
206
|
+
|
207
|
+
initialize_logger
|
208
|
+
end
|
209
|
+
|
210
|
+
def set_environment(cli_env)
|
211
|
+
@environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
212
|
+
end
|
213
|
+
|
214
|
+
alias_method :die, :exit
|
215
|
+
alias_method :☠, :exit
|
216
|
+
|
217
|
+
def setup_options(args)
|
218
|
+
opts = parse_options(args)
|
219
|
+
set_environment opts[:environment]
|
220
|
+
|
221
|
+
cfile = opts[:config_file]
|
222
|
+
opts = parse_config(cfile).merge(opts) if cfile
|
223
|
+
|
224
|
+
opts[:strict] = true if opts[:strict].nil?
|
225
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
|
226
|
+
opts[:identity] = identity
|
227
|
+
|
228
|
+
options.merge!(opts)
|
229
|
+
end
|
230
|
+
|
231
|
+
def options
|
232
|
+
Sidekiq.options
|
233
|
+
end
|
234
|
+
|
235
|
+
def boot_system
|
236
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
|
237
|
+
|
238
|
+
raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
|
239
|
+
|
240
|
+
if File.directory?(options[:require])
|
241
|
+
require 'rails'
|
242
|
+
if ::Rails::VERSION::MAJOR < 4
|
243
|
+
raise "Sidekiq no longer supports this version of Rails"
|
244
|
+
elsif ::Rails::VERSION::MAJOR == 4
|
245
|
+
# Painful contortions, see 1791 for discussion
|
246
|
+
# No autoloading, we want to force eager load for everything.
|
247
|
+
require File.expand_path("#{options[:require]}/config/application.rb")
|
248
|
+
::Rails::Application.initializer "sidekiq.eager_load" do
|
249
|
+
::Rails.application.config.eager_load = true
|
250
|
+
end
|
251
|
+
require 'sidekiq/rails'
|
252
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
253
|
+
else
|
254
|
+
require 'sidekiq/rails'
|
255
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
256
|
+
end
|
257
|
+
options[:tag] ||= default_tag
|
258
|
+
else
|
259
|
+
not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
|
260
|
+
"./#{options[:require]} or /path/to/#{options[:require]}"
|
261
|
+
|
262
|
+
require(options[:require]) || raise(ArgumentError, not_required_message)
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
def default_tag
|
267
|
+
dir = ::Rails.root
|
268
|
+
name = File.basename(dir)
|
269
|
+
if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
|
270
|
+
if File.basename(prevdir) == 'releases'
|
271
|
+
return File.basename(File.dirname(prevdir))
|
272
|
+
end
|
273
|
+
end
|
274
|
+
name
|
275
|
+
end
|
276
|
+
|
277
|
+
def validate!
|
278
|
+
options[:queues] << 'default' if options[:queues].empty?
|
279
|
+
|
280
|
+
if !File.exist?(options[:require]) ||
|
281
|
+
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
282
|
+
logger.info "=================================================================="
|
283
|
+
logger.info " Please point sidekiq to a Rails 3/4 application or a Ruby file "
|
284
|
+
logger.info " to load your worker classes with -r [DIR|FILE]."
|
285
|
+
logger.info "=================================================================="
|
286
|
+
logger.info @parser
|
287
|
+
die(1)
|
288
|
+
end
|
289
|
+
|
290
|
+
[:concurrency, :timeout].each do |opt|
|
291
|
+
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
def parse_options(argv)
|
296
|
+
opts = {}
|
297
|
+
|
298
|
+
@parser = OptionParser.new do |o|
|
299
|
+
o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
|
300
|
+
opts[:concurrency] = Integer(arg)
|
301
|
+
end
|
302
|
+
|
303
|
+
o.on '-d', '--daemon', "Daemonize process" do |arg|
|
304
|
+
opts[:daemon] = arg
|
305
|
+
end
|
306
|
+
|
307
|
+
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
308
|
+
opts[:environment] = arg
|
309
|
+
end
|
310
|
+
|
311
|
+
o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
|
312
|
+
opts[:tag] = arg
|
313
|
+
end
|
314
|
+
|
315
|
+
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
316
|
+
opts[:index] = Integer(arg.match(/\d+/)[0])
|
317
|
+
end
|
318
|
+
|
319
|
+
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
320
|
+
queue, weight = arg.split(",")
|
321
|
+
parse_queue opts, queue, weight
|
322
|
+
end
|
323
|
+
|
324
|
+
o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
|
325
|
+
opts[:require] = arg
|
326
|
+
end
|
327
|
+
|
328
|
+
o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
|
329
|
+
opts[:timeout] = Integer(arg)
|
330
|
+
end
|
331
|
+
|
332
|
+
o.on "-v", "--verbose", "Print more verbose output" do |arg|
|
333
|
+
opts[:verbose] = arg
|
334
|
+
end
|
335
|
+
|
336
|
+
o.on '-C', '--config PATH', "path to YAML config file" do |arg|
|
337
|
+
opts[:config_file] = arg
|
338
|
+
end
|
339
|
+
|
340
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
|
341
|
+
opts[:logfile] = arg
|
342
|
+
end
|
343
|
+
|
344
|
+
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
345
|
+
opts[:pidfile] = arg
|
346
|
+
end
|
347
|
+
|
348
|
+
o.on '-V', '--version', "Print version and exit" do |arg|
|
349
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
350
|
+
die(0)
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
@parser.banner = "sidekiq [options]"
|
355
|
+
@parser.on_tail "-h", "--help", "Show help" do
|
356
|
+
logger.info @parser
|
357
|
+
die 1
|
358
|
+
end
|
359
|
+
@parser.parse!(argv)
|
360
|
+
|
361
|
+
%w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
|
362
|
+
opts[:config_file] ||= filename if File.exist?(filename)
|
363
|
+
end
|
364
|
+
|
365
|
+
opts
|
366
|
+
end
|
367
|
+
|
368
|
+
def initialize_logger
|
369
|
+
Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
|
370
|
+
|
371
|
+
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
372
|
+
end
|
373
|
+
|
374
|
+
def write_pid
|
375
|
+
if path = options[:pidfile]
|
376
|
+
pidfile = File.expand_path(path)
|
377
|
+
File.open(pidfile, 'w') do |f|
|
378
|
+
f.puts ::Process.pid
|
379
|
+
end
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
def parse_config(cfile)
|
384
|
+
opts = {}
|
385
|
+
if File.exist?(cfile)
|
386
|
+
opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
|
387
|
+
opts = opts.merge(opts.delete(environment) || {})
|
388
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
389
|
+
else
|
390
|
+
# allow a non-existent config file so Sidekiq
|
391
|
+
# can be deployed by cap with just the defaults.
|
392
|
+
end
|
393
|
+
ns = opts.delete(:namespace)
|
394
|
+
if ns
|
395
|
+
# logger hasn't been initialized yet, puts is all we have.
|
396
|
+
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
397
|
+
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
398
|
+
end
|
399
|
+
opts
|
400
|
+
end
|
401
|
+
|
402
|
+
def parse_queues(opts, queues_and_weights)
|
403
|
+
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
404
|
+
end
|
405
|
+
|
406
|
+
def parse_queue(opts, q, weight=nil)
|
407
|
+
[weight.to_i, 1].max.times do
|
408
|
+
(opts[:queues] ||= []) << q
|
409
|
+
end
|
410
|
+
opts[:strict] = false if weight.to_i > 0
|
411
|
+
end
|
412
|
+
end
|
413
|
+
end
|
@@ -0,0 +1,238 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'securerandom'
|
3
|
+
require 'sidekiq/middleware/chain'
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
class Client
|
7
|
+
|
8
|
+
##
|
9
|
+
# Define client-side middleware:
|
10
|
+
#
|
11
|
+
# client = Sidekiq::Client.new
|
12
|
+
# client.middleware do |chain|
|
13
|
+
# chain.use MyClientMiddleware
|
14
|
+
# end
|
15
|
+
# client.push('class' => 'SomeWorker', 'args' => [1,2,3])
|
16
|
+
#
|
17
|
+
# All client instances default to the globally-defined
|
18
|
+
# Sidekiq.client_middleware but you can change as necessary.
|
19
|
+
#
|
20
|
+
def middleware(&block)
|
21
|
+
@chain ||= Sidekiq.client_middleware
|
22
|
+
if block_given?
|
23
|
+
@chain = @chain.dup
|
24
|
+
yield @chain
|
25
|
+
end
|
26
|
+
@chain
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_accessor :redis_pool
|
30
|
+
|
31
|
+
# Sidekiq::Client normally uses the default Redis pool but you may
|
32
|
+
# pass a custom ConnectionPool if you want to shard your
|
33
|
+
# Sidekiq jobs across several Redis instances (for scalability
|
34
|
+
# reasons, e.g.)
|
35
|
+
#
|
36
|
+
# Sidekiq::Client.new(ConnectionPool.new { Redis.new })
|
37
|
+
#
|
38
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
39
|
+
# thousands of jobs per second. I don't recommend sharding unless you
|
40
|
+
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
+
def initialize(redis_pool=nil)
|
42
|
+
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
|
+
end
|
44
|
+
|
45
|
+
##
|
46
|
+
# The main method used to push a job to Redis. Accepts a number of options:
|
47
|
+
#
|
48
|
+
# queue - the named queue to use, default 'default'
|
49
|
+
# class - the worker class to call, required
|
50
|
+
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
|
+
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
|
+
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
|
+
# backtrace - whether to save any error backtrace, default false
|
54
|
+
#
|
55
|
+
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
56
|
+
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
57
|
+
# space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
|
58
|
+
#
|
59
|
+
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
60
|
+
#
|
61
|
+
# Example:
|
62
|
+
# push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
|
63
|
+
#
|
64
|
+
def push(item)
|
65
|
+
normed = normalize_item(item)
|
66
|
+
payload = process_single(item['class'.freeze], normed)
|
67
|
+
|
68
|
+
if payload
|
69
|
+
raw_push([payload])
|
70
|
+
payload['jid'.freeze]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
##
|
75
|
+
# Push a large number of jobs to Redis. In practice this method is only
|
76
|
+
# useful if you are pushing thousands of jobs or more. This method
|
77
|
+
# cuts out the redis network round trip latency.
|
78
|
+
#
|
79
|
+
# Takes the same arguments as #push except that args is expected to be
|
80
|
+
# an Array of Arrays. All other keys are duplicated for each job. Each job
|
81
|
+
# is run through the client middleware pipeline and each job gets its own Job ID
|
82
|
+
# as normal.
|
83
|
+
#
|
84
|
+
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
85
|
+
# than the number given if the middleware stopped processing for one or more jobs.
|
86
|
+
def push_bulk(items)
|
87
|
+
arg = items['args'.freeze].first
|
88
|
+
return [] unless arg # no jobs to push
|
89
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
|
90
|
+
|
91
|
+
normed = normalize_item(items)
|
92
|
+
payloads = items['args'.freeze].map do |args|
|
93
|
+
copy = normed.merge('args'.freeze => args, 'jid'.freeze => SecureRandom.hex(12), 'enqueued_at'.freeze => Time.now.to_f)
|
94
|
+
result = process_single(items['class'.freeze], copy)
|
95
|
+
result ? result : nil
|
96
|
+
end.compact
|
97
|
+
|
98
|
+
raw_push(payloads) if !payloads.empty?
|
99
|
+
payloads.collect { |payload| payload['jid'.freeze] }
|
100
|
+
end
|
101
|
+
|
102
|
+
# Allows sharding of jobs across any number of Redis instances. All jobs
|
103
|
+
# defined within the block will use the given Redis connection pool.
|
104
|
+
#
|
105
|
+
# pool = ConnectionPool.new { Redis.new }
|
106
|
+
# Sidekiq::Client.via(pool) do
|
107
|
+
# SomeWorker.perform_async(1,2,3)
|
108
|
+
# SomeOtherWorker.perform_async(1,2,3)
|
109
|
+
# end
|
110
|
+
#
|
111
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
112
|
+
# thousands of jobs per second. I do not recommend sharding unless
|
113
|
+
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
114
|
+
def self.via(pool)
|
115
|
+
raise ArgumentError, "No pool given" if pool.nil?
|
116
|
+
current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
|
117
|
+
raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
|
118
|
+
Thread.current[:sidekiq_via_pool] = pool
|
119
|
+
yield
|
120
|
+
ensure
|
121
|
+
Thread.current[:sidekiq_via_pool] = nil
|
122
|
+
end
|
123
|
+
|
124
|
+
class << self
|
125
|
+
|
126
|
+
def push(item)
|
127
|
+
new.push(item)
|
128
|
+
end
|
129
|
+
|
130
|
+
def push_bulk(items)
|
131
|
+
new.push_bulk(items)
|
132
|
+
end
|
133
|
+
|
134
|
+
# Resque compatibility helpers. Note all helpers
|
135
|
+
# should go through Worker#client_push.
|
136
|
+
#
|
137
|
+
# Example usage:
|
138
|
+
# Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
|
139
|
+
#
|
140
|
+
# Messages are enqueued to the 'default' queue.
|
141
|
+
#
|
142
|
+
def enqueue(klass, *args)
|
143
|
+
klass.client_push('class'.freeze => klass, 'args'.freeze => args)
|
144
|
+
end
|
145
|
+
|
146
|
+
# Example usage:
|
147
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
148
|
+
#
|
149
|
+
def enqueue_to(queue, klass, *args)
|
150
|
+
klass.client_push('queue'.freeze => queue, 'class'.freeze => klass, 'args'.freeze => args)
|
151
|
+
end
|
152
|
+
|
153
|
+
# Example usage:
|
154
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
155
|
+
#
|
156
|
+
def enqueue_to_in(queue, interval, klass, *args)
|
157
|
+
int = interval.to_f
|
158
|
+
now = Time.now.to_f
|
159
|
+
ts = (int < 1_000_000_000 ? now + int : int)
|
160
|
+
|
161
|
+
item = { 'class'.freeze => klass, 'args'.freeze => args, 'at'.freeze => ts, 'queue'.freeze => queue }
|
162
|
+
item.delete('at'.freeze) if ts <= now
|
163
|
+
|
164
|
+
klass.client_push(item)
|
165
|
+
end
|
166
|
+
|
167
|
+
# Example usage:
|
168
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
169
|
+
#
|
170
|
+
def enqueue_in(interval, klass, *args)
|
171
|
+
klass.perform_in(interval, *args)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
private
|
176
|
+
|
177
|
+
def raw_push(payloads)
|
178
|
+
@redis_pool.with do |conn|
|
179
|
+
conn.multi do
|
180
|
+
atomic_push(conn, payloads)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
true
|
184
|
+
end
|
185
|
+
|
186
|
+
def atomic_push(conn, payloads)
|
187
|
+
if payloads.first['at'.freeze]
|
188
|
+
conn.zadd('schedule'.freeze, payloads.map do |hash|
|
189
|
+
at = hash.delete('at'.freeze).to_s
|
190
|
+
[at, Sidekiq.dump_json(hash)]
|
191
|
+
end)
|
192
|
+
else
|
193
|
+
q = payloads.first['queue'.freeze]
|
194
|
+
now = Time.now.to_f
|
195
|
+
to_push = payloads.map do |entry|
|
196
|
+
entry['enqueued_at'.freeze] = now
|
197
|
+
Sidekiq.dump_json(entry)
|
198
|
+
end
|
199
|
+
conn.sadd('queues'.freeze, q)
|
200
|
+
conn.lpush("queue:#{q}", to_push)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def process_single(worker_class, item)
|
205
|
+
queue = item['queue'.freeze]
|
206
|
+
|
207
|
+
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
208
|
+
item
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def normalize_item(item)
|
213
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
|
214
|
+
raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
|
215
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
|
216
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at'.freeze) && !item['at'].is_a?(Numeric)
|
217
|
+
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
218
|
+
|
219
|
+
normalized_hash(item['class'.freeze])
|
220
|
+
.each{ |key, value| item[key] = value if item[key].nil? }
|
221
|
+
|
222
|
+
item['class'.freeze] = item['class'.freeze].to_s
|
223
|
+
item['queue'.freeze] = item['queue'.freeze].to_s
|
224
|
+
item['jid'.freeze] ||= SecureRandom.hex(12)
|
225
|
+
item['created_at'.freeze] ||= Time.now.to_f
|
226
|
+
item
|
227
|
+
end
|
228
|
+
|
229
|
+
def normalized_hash(item_class)
|
230
|
+
if item_class.is_a?(Class)
|
231
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
|
232
|
+
item_class.get_sidekiq_options
|
233
|
+
else
|
234
|
+
Sidekiq.default_worker_options
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|