sidekiq_cleaner 5.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +61 -0
- data/.github/contributing.md +32 -0
- data/.github/issue_template.md +11 -0
- data/.gitignore +15 -0
- data/.travis.yml +11 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +53 -0
- data/5.0-Upgrade.md +56 -0
- data/COMM-LICENSE +97 -0
- data/Changes.md +1536 -0
- data/Ent-Changes.md +238 -0
- data/Gemfile +23 -0
- data/LICENSE +9 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-4.0-Upgrade.md +35 -0
- data/Pro-Changes.md +759 -0
- data/README.md +55 -0
- data/Rakefile +9 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +20 -0
- data/bin/sidekiqload +149 -0
- data/cleaner/assets/images/favicon.ico +0 -0
- data/cleaner/assets/images/logo.png +0 -0
- data/cleaner/assets/images/status.png +0 -0
- data/cleaner/assets/javascripts/application.js +172 -0
- data/cleaner/assets/javascripts/dashboard.js +315 -0
- data/cleaner/assets/stylesheets/application-rtl.css +246 -0
- data/cleaner/assets/stylesheets/application.css +1144 -0
- data/cleaner/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/cleaner/assets/stylesheets/bootstrap.css +5 -0
- data/cleaner/locales/ar.yml +81 -0
- data/cleaner/locales/cs.yml +78 -0
- data/cleaner/locales/da.yml +68 -0
- data/cleaner/locales/de.yml +69 -0
- data/cleaner/locales/el.yml +68 -0
- data/cleaner/locales/en.yml +81 -0
- data/cleaner/locales/es.yml +70 -0
- data/cleaner/locales/fa.yml +80 -0
- data/cleaner/locales/fr.yml +78 -0
- data/cleaner/locales/he.yml +79 -0
- data/cleaner/locales/hi.yml +75 -0
- data/cleaner/locales/it.yml +69 -0
- data/cleaner/locales/ja.yml +80 -0
- data/cleaner/locales/ko.yml +68 -0
- data/cleaner/locales/nb.yml +77 -0
- data/cleaner/locales/nl.yml +68 -0
- data/cleaner/locales/pl.yml +59 -0
- data/cleaner/locales/pt-br.yml +68 -0
- data/cleaner/locales/pt.yml +67 -0
- data/cleaner/locales/ru.yml +78 -0
- data/cleaner/locales/sv.yml +68 -0
- data/cleaner/locales/ta.yml +75 -0
- data/cleaner/locales/uk.yml +76 -0
- data/cleaner/locales/ur.yml +80 -0
- data/cleaner/locales/zh-cn.yml +68 -0
- data/cleaner/locales/zh-tw.yml +68 -0
- data/cleaner/views/_footer.erb +20 -0
- data/cleaner/views/_job_info.erb +88 -0
- data/cleaner/views/_nav.erb +52 -0
- data/cleaner/views/_paging.erb +23 -0
- data/cleaner/views/_poll_link.erb +7 -0
- data/cleaner/views/_status.erb +4 -0
- data/cleaner/views/_summary.erb +40 -0
- data/cleaner/views/busy.erb +98 -0
- data/cleaner/views/dashboard.erb +75 -0
- data/cleaner/views/dead.erb +34 -0
- data/cleaner/views/errors.erb +84 -0
- data/cleaner/views/layout.erb +40 -0
- data/cleaner/views/morgue.erb +75 -0
- data/cleaner/views/queue.erb +46 -0
- data/cleaner/views/queues.erb +30 -0
- data/cleaner/views/retries.erb +80 -0
- data/cleaner/views/retry.erb +34 -0
- data/cleaner/views/scheduled.erb +54 -0
- data/cleaner/views/scheduled_job_info.erb +8 -0
- data/cleaner-stats.png +0 -0
- data/cleaner.png +0 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq/api.rb +940 -0
- data/lib/sidekiq/cleaner/action.rb +89 -0
- data/lib/sidekiq/cleaner/application.rb +385 -0
- data/lib/sidekiq/cleaner/helpers.rb +325 -0
- data/lib/sidekiq/cleaner/router.rb +100 -0
- data/lib/sidekiq/cleaner.rb +214 -0
- data/lib/sidekiq/cli.rb +445 -0
- data/lib/sidekiq/client.rb +243 -0
- data/lib/sidekiq/core_ext.rb +1 -0
- data/lib/sidekiq/ctl.rb +221 -0
- data/lib/sidekiq/delay.rb +42 -0
- data/lib/sidekiq/exception_handler.rb +29 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/job_logger.rb +25 -0
- data/lib/sidekiq/job_retry.rb +262 -0
- data/lib/sidekiq/launcher.rb +173 -0
- data/lib/sidekiq/logging.rb +122 -0
- data/lib/sidekiq/manager.rb +137 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +23 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +279 -0
- data/lib/sidekiq/rails.rb +58 -0
- data/lib/sidekiq/redis_connection.rb +144 -0
- data/lib/sidekiq/scheduled.rb +174 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/testing.rb +333 -0
- data/lib/sidekiq/util.rb +66 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/worker.rb +220 -0
- data/lib/sidekiq.rb +237 -0
- data/sidekiq_cleaner.gemspec +21 -0
- metadata +235 -0
data/lib/sidekiq/cli.rb
ADDED
@@ -0,0 +1,445 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
$stdout.sync = true
|
3
|
+
|
4
|
+
require 'yaml'
|
5
|
+
require 'singleton'
|
6
|
+
require 'optparse'
|
7
|
+
require 'erb'
|
8
|
+
require 'fileutils'
|
9
|
+
|
10
|
+
require 'sidekiq'
|
11
|
+
require 'sidekiq/util'
|
12
|
+
require 'sidekiq/launcher'
|
13
|
+
|
14
|
+
module Sidekiq
|
15
|
+
class CLI
|
16
|
+
include Util
|
17
|
+
include Singleton unless $TESTING
|
18
|
+
|
19
|
+
PROCTITLES = [
|
20
|
+
proc { 'sidekiq' },
|
21
|
+
proc { Sidekiq::VERSION },
|
22
|
+
proc { |me, data| data['tag'] },
|
23
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
+
proc { |me, data| "stopping" if me.stopping? },
|
25
|
+
]
|
26
|
+
|
27
|
+
attr_accessor :launcher
|
28
|
+
attr_accessor :environment
|
29
|
+
|
30
|
+
def parse(args = ARGV)
|
31
|
+
setup_options(args)
|
32
|
+
initialize_logger
|
33
|
+
validate!
|
34
|
+
end
|
35
|
+
|
36
|
+
def jruby?
|
37
|
+
defined?(::JRUBY_VERSION)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Code within this method is not tested because it alters
|
41
|
+
# global process state irreversibly. PRs which improve the
|
42
|
+
# test coverage of Sidekiq::CLI are welcomed.
|
43
|
+
def run
|
44
|
+
daemonize if options[:daemon]
|
45
|
+
write_pid
|
46
|
+
boot_system
|
47
|
+
print_banner if environment == 'development' && $stdout.tty?
|
48
|
+
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w(INT TERM TTIN TSTP)
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
if !jruby?
|
53
|
+
sigs << 'USR1'
|
54
|
+
sigs << 'USR2'
|
55
|
+
end
|
56
|
+
|
57
|
+
sigs.each do |sig|
|
58
|
+
begin
|
59
|
+
trap sig do
|
60
|
+
self_write.write("#{sig}\n")
|
61
|
+
end
|
62
|
+
rescue ArgumentError
|
63
|
+
puts "Signal #{sig} not supported"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
68
|
+
logger.info Sidekiq::LICENSE
|
69
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
70
|
+
|
71
|
+
# touch the connection pool so it is created before we
|
72
|
+
# fire startup and start multithreading.
|
73
|
+
ver = Sidekiq.redis_info['redis_version']
|
74
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
|
75
|
+
logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
|
76
|
+
|
77
|
+
# Since the user can pass us a connection pool explicitly in the initializer, we
|
78
|
+
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
79
|
+
cursize = Sidekiq.redis_pool.size
|
80
|
+
needed = Sidekiq.options[:concurrency] + 2
|
81
|
+
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
82
|
+
|
83
|
+
# cache process identity
|
84
|
+
Sidekiq.options[:identity] = identity
|
85
|
+
|
86
|
+
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
87
|
+
Sidekiq.server_middleware
|
88
|
+
|
89
|
+
# Before this point, the process is initializing with just the main thread.
|
90
|
+
# Starting here the process will now have multiple threads running.
|
91
|
+
fire_event(:startup, reverse: false, reraise: true)
|
92
|
+
|
93
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
|
94
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
|
95
|
+
|
96
|
+
launch(self_read)
|
97
|
+
end
|
98
|
+
|
99
|
+
def launch(self_read)
|
100
|
+
if !options[:daemon]
|
101
|
+
logger.info 'Starting processing, hit Ctrl-C to stop'
|
102
|
+
end
|
103
|
+
|
104
|
+
@launcher = Sidekiq::Launcher.new(options)
|
105
|
+
|
106
|
+
begin
|
107
|
+
launcher.run
|
108
|
+
|
109
|
+
while readable_io = IO.select([self_read])
|
110
|
+
signal = readable_io.first[0].gets.strip
|
111
|
+
handle_signal(signal)
|
112
|
+
end
|
113
|
+
rescue Interrupt
|
114
|
+
logger.info 'Shutting down'
|
115
|
+
launcher.stop
|
116
|
+
# Explicitly exit so busy Processor threads can't block
|
117
|
+
# process shutdown.
|
118
|
+
logger.info "Bye!"
|
119
|
+
exit(0)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.banner
|
124
|
+
%q{
|
125
|
+
m,
|
126
|
+
`$b
|
127
|
+
.ss, $$: .,d$
|
128
|
+
`$$P,d$P' .,md$P"'
|
129
|
+
,$$$$$bmmd$$$P^'
|
130
|
+
.d$$$$$$$$$$P'
|
131
|
+
$$^' `"^$$$' ____ _ _ _ _
|
132
|
+
$: ,$$: / ___|(_) __| | ___| | _(_) __ _
|
133
|
+
`b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
|
134
|
+
$$: ___) | | (_| | __/ <| | (_| |
|
135
|
+
$$ |____/|_|\__,_|\___|_|\_\_|\__, |
|
136
|
+
.d$$ |_|
|
137
|
+
}
|
138
|
+
end
|
139
|
+
|
140
|
+
SIGNAL_HANDLERS = {
|
141
|
+
# Ctrl-C in terminal
|
142
|
+
'INT' => ->(cli) { raise Interrupt },
|
143
|
+
# TERM is the signal that Sidekiq must exit.
|
144
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
145
|
+
'TERM' => ->(cli) { raise Interrupt },
|
146
|
+
'USR1' => ->(cli) {
|
147
|
+
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
148
|
+
cli.launcher.quiet
|
149
|
+
},
|
150
|
+
'TSTP' => ->(cli) {
|
151
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
152
|
+
cli.launcher.quiet
|
153
|
+
},
|
154
|
+
'USR2' => ->(cli) {
|
155
|
+
if Sidekiq.options[:logfile]
|
156
|
+
Sidekiq.logger.info "Received USR2, reopening log file"
|
157
|
+
Sidekiq::Logging.reopen_logs
|
158
|
+
end
|
159
|
+
},
|
160
|
+
'TTIN' => ->(cli) {
|
161
|
+
Thread.list.each do |thread|
|
162
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
|
163
|
+
if thread.backtrace
|
164
|
+
Sidekiq.logger.warn thread.backtrace.join("\n")
|
165
|
+
else
|
166
|
+
Sidekiq.logger.warn "<no backtrace available>"
|
167
|
+
end
|
168
|
+
end
|
169
|
+
},
|
170
|
+
}
|
171
|
+
|
172
|
+
def handle_signal(sig)
|
173
|
+
Sidekiq.logger.debug "Got #{sig} signal"
|
174
|
+
handy = SIGNAL_HANDLERS[sig]
|
175
|
+
if handy
|
176
|
+
handy.call(self)
|
177
|
+
else
|
178
|
+
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
private
|
183
|
+
|
184
|
+
def print_banner
|
185
|
+
puts "\e[#{31}m"
|
186
|
+
puts Sidekiq::CLI.banner
|
187
|
+
puts "\e[0m"
|
188
|
+
end
|
189
|
+
|
190
|
+
def daemonize
|
191
|
+
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
192
|
+
|
193
|
+
files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
|
194
|
+
::Process.daemon(true, true)
|
195
|
+
|
196
|
+
files_to_reopen.each do |file|
|
197
|
+
begin
|
198
|
+
file.reopen file.path, "a+"
|
199
|
+
file.sync = true
|
200
|
+
rescue ::Exception
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
[$stdout, $stderr].each do |io|
|
205
|
+
File.open(options[:logfile], 'ab') do |f|
|
206
|
+
io.reopen(f)
|
207
|
+
end
|
208
|
+
io.sync = true
|
209
|
+
end
|
210
|
+
$stdin.reopen('/dev/null')
|
211
|
+
|
212
|
+
initialize_logger
|
213
|
+
end
|
214
|
+
|
215
|
+
def set_environment(cli_env)
|
216
|
+
@environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
217
|
+
end
|
218
|
+
|
219
|
+
def symbolize_keys_deep!(hash)
|
220
|
+
hash.keys.each do |k|
|
221
|
+
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
222
|
+
hash[symkey] = hash.delete k
|
223
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
alias_method :die, :exit
|
228
|
+
alias_method :☠, :exit
|
229
|
+
|
230
|
+
def setup_options(args)
|
231
|
+
# parse CLI options
|
232
|
+
opts = parse_options(args)
|
233
|
+
|
234
|
+
set_environment opts[:environment]
|
235
|
+
|
236
|
+
# check config file presence
|
237
|
+
if opts[:config_file]
|
238
|
+
if opts[:config_file] && !File.exist?(opts[:config_file])
|
239
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
240
|
+
end
|
241
|
+
else
|
242
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
243
|
+
File.join(opts[:require], 'config')
|
244
|
+
else
|
245
|
+
File.join(options[:require], 'config')
|
246
|
+
end
|
247
|
+
|
248
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
249
|
+
path = File.join(config_dir, config_file)
|
250
|
+
opts[:config_file] ||= path if File.exist?(path)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# parse config file options
|
255
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
256
|
+
|
257
|
+
# set defaults
|
258
|
+
opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
|
259
|
+
opts[:strict] = true if opts[:strict].nil?
|
260
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
261
|
+
|
262
|
+
# merge with defaults
|
263
|
+
options.merge!(opts)
|
264
|
+
end
|
265
|
+
|
266
|
+
def options
|
267
|
+
Sidekiq.options
|
268
|
+
end
|
269
|
+
|
270
|
+
def boot_system
|
271
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
|
272
|
+
|
273
|
+
if File.directory?(options[:require])
|
274
|
+
require 'rails'
|
275
|
+
if ::Rails::VERSION::MAJOR < 4
|
276
|
+
raise "Sidekiq no longer supports this version of Rails"
|
277
|
+
elsif ::Rails::VERSION::MAJOR == 4
|
278
|
+
# Painful contortions, see 1791 for discussion
|
279
|
+
# No autoloading, we want to force eager load for everything.
|
280
|
+
require File.expand_path("#{options[:require]}/config/application.rb")
|
281
|
+
::Rails::Application.initializer "sidekiq.eager_load" do
|
282
|
+
::Rails.application.config.eager_load = true
|
283
|
+
end
|
284
|
+
require 'sidekiq/rails'
|
285
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
286
|
+
else
|
287
|
+
require 'sidekiq/rails'
|
288
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
289
|
+
end
|
290
|
+
options[:tag] ||= default_tag
|
291
|
+
else
|
292
|
+
require options[:require]
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
def default_tag
|
297
|
+
dir = ::Rails.root
|
298
|
+
name = File.basename(dir)
|
299
|
+
if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
|
300
|
+
if File.basename(prevdir) == 'releases'
|
301
|
+
return File.basename(File.dirname(prevdir))
|
302
|
+
end
|
303
|
+
end
|
304
|
+
name
|
305
|
+
end
|
306
|
+
|
307
|
+
def validate!
|
308
|
+
if !File.exist?(options[:require]) ||
|
309
|
+
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
310
|
+
logger.info "=================================================================="
|
311
|
+
logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
|
312
|
+
logger.info " to load your worker classes with -r [DIR|FILE]."
|
313
|
+
logger.info "=================================================================="
|
314
|
+
logger.info @parser
|
315
|
+
die(1)
|
316
|
+
end
|
317
|
+
|
318
|
+
[:concurrency, :timeout].each do |opt|
|
319
|
+
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
def parse_options(argv)
|
324
|
+
opts = {}
|
325
|
+
|
326
|
+
@parser = OptionParser.new do |o|
|
327
|
+
o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
|
328
|
+
opts[:concurrency] = Integer(arg)
|
329
|
+
end
|
330
|
+
|
331
|
+
o.on '-d', '--daemon', "Daemonize process" do |arg|
|
332
|
+
opts[:daemon] = arg
|
333
|
+
puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
334
|
+
end
|
335
|
+
|
336
|
+
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
337
|
+
opts[:environment] = arg
|
338
|
+
end
|
339
|
+
|
340
|
+
o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
|
341
|
+
opts[:tag] = arg
|
342
|
+
end
|
343
|
+
|
344
|
+
# this index remains here for backwards compatibility but none of the Sidekiq
|
345
|
+
# family use this value anymore. it was used by Pro's original reliable_fetch.
|
346
|
+
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
347
|
+
opts[:index] = Integer(arg.match(/\d+/)[0])
|
348
|
+
end
|
349
|
+
|
350
|
+
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
351
|
+
queue, weight = arg.split(",")
|
352
|
+
parse_queue opts, queue, weight
|
353
|
+
end
|
354
|
+
|
355
|
+
o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
|
356
|
+
opts[:require] = arg
|
357
|
+
end
|
358
|
+
|
359
|
+
o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
|
360
|
+
opts[:timeout] = Integer(arg)
|
361
|
+
end
|
362
|
+
|
363
|
+
o.on "-v", "--verbose", "Print more verbose output" do |arg|
|
364
|
+
opts[:verbose] = arg
|
365
|
+
end
|
366
|
+
|
367
|
+
o.on '-C', '--config PATH', "path to YAML config file" do |arg|
|
368
|
+
opts[:config_file] = arg
|
369
|
+
end
|
370
|
+
|
371
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
|
372
|
+
opts[:logfile] = arg
|
373
|
+
puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
|
374
|
+
end
|
375
|
+
|
376
|
+
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
377
|
+
opts[:pidfile] = arg
|
378
|
+
puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
379
|
+
end
|
380
|
+
|
381
|
+
o.on '-V', '--version', "Print version and exit" do |arg|
|
382
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
383
|
+
die(0)
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
@parser.banner = "sidekiq [options]"
|
388
|
+
@parser.on_tail "-h", "--help", "Show help" do
|
389
|
+
logger.info @parser
|
390
|
+
die 1
|
391
|
+
end
|
392
|
+
|
393
|
+
@parser.parse!(argv)
|
394
|
+
|
395
|
+
opts
|
396
|
+
end
|
397
|
+
|
398
|
+
def initialize_logger
|
399
|
+
Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
|
400
|
+
|
401
|
+
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
402
|
+
end
|
403
|
+
|
404
|
+
def write_pid
|
405
|
+
if path = options[:pidfile]
|
406
|
+
pidfile = File.expand_path(path)
|
407
|
+
File.open(pidfile, 'w') do |f|
|
408
|
+
f.puts ::Process.pid
|
409
|
+
end
|
410
|
+
end
|
411
|
+
end
|
412
|
+
|
413
|
+
def parse_config(path)
|
414
|
+
opts = YAML.load(ERB.new(File.read(path)).result) || {}
|
415
|
+
|
416
|
+
if opts.respond_to? :deep_symbolize_keys!
|
417
|
+
opts.deep_symbolize_keys!
|
418
|
+
else
|
419
|
+
symbolize_keys_deep!(opts)
|
420
|
+
end
|
421
|
+
|
422
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
423
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
424
|
+
|
425
|
+
ns = opts.delete(:namespace)
|
426
|
+
if ns
|
427
|
+
# logger hasn't been initialized yet, puts is all we have.
|
428
|
+
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
429
|
+
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
430
|
+
end
|
431
|
+
opts
|
432
|
+
end
|
433
|
+
|
434
|
+
def parse_queues(opts, queues_and_weights)
|
435
|
+
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
436
|
+
end
|
437
|
+
|
438
|
+
def parse_queue(opts, queue, weight = nil)
|
439
|
+
opts[:queues] ||= []
|
440
|
+
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
441
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue }
|
442
|
+
opts[:strict] = false if weight.to_i > 0
|
443
|
+
end
|
444
|
+
end
|
445
|
+
end
|
@@ -0,0 +1,243 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'securerandom'
|
3
|
+
require 'sidekiq/middleware/chain'
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
class Client
|
7
|
+
|
8
|
+
##
|
9
|
+
# Define client-side middleware:
|
10
|
+
#
|
11
|
+
# client = Sidekiq::Client.new
|
12
|
+
# client.middleware do |chain|
|
13
|
+
# chain.use MyClientMiddleware
|
14
|
+
# end
|
15
|
+
# client.push('class' => 'SomeWorker', 'args' => [1,2,3])
|
16
|
+
#
|
17
|
+
# All client instances default to the globally-defined
|
18
|
+
# Sidekiq.client_middleware but you can change as necessary.
|
19
|
+
#
|
20
|
+
def middleware(&block)
|
21
|
+
@chain ||= Sidekiq.client_middleware
|
22
|
+
if block_given?
|
23
|
+
@chain = @chain.dup
|
24
|
+
yield @chain
|
25
|
+
end
|
26
|
+
@chain
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_accessor :redis_pool
|
30
|
+
|
31
|
+
# Sidekiq::Client normally uses the default Redis pool but you may
|
32
|
+
# pass a custom ConnectionPool if you want to shard your
|
33
|
+
# Sidekiq jobs across several Redis instances (for scalability
|
34
|
+
# reasons, e.g.)
|
35
|
+
#
|
36
|
+
# Sidekiq::Client.new(ConnectionPool.new { Redis.new })
|
37
|
+
#
|
38
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
39
|
+
# thousands of jobs per second. I don't recommend sharding unless you
|
40
|
+
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
+
def initialize(redis_pool=nil)
|
42
|
+
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
|
+
end
|
44
|
+
|
45
|
+
##
|
46
|
+
# The main method used to push a job to Redis. Accepts a number of options:
|
47
|
+
#
|
48
|
+
# queue - the named queue to use, default 'default'
|
49
|
+
# class - the worker class to call, required
|
50
|
+
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
|
+
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
|
+
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
|
+
# backtrace - whether to save any error backtrace, default false
|
54
|
+
#
|
55
|
+
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
+
# worker options. Otherwise, they will be based on the job class's options.
|
57
|
+
#
|
58
|
+
# Any options valid for a worker class's sidekiq_options are also available here.
|
59
|
+
#
|
60
|
+
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
61
|
+
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
62
|
+
# space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
|
63
|
+
#
|
64
|
+
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
65
|
+
#
|
66
|
+
# Example:
|
67
|
+
# push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
|
68
|
+
#
|
69
|
+
def push(item)
|
70
|
+
normed = normalize_item(item)
|
71
|
+
payload = process_single(item['class'], normed)
|
72
|
+
|
73
|
+
if payload
|
74
|
+
raw_push([payload])
|
75
|
+
payload['jid']
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
##
|
80
|
+
# Push a large number of jobs to Redis. This method cuts out the redis
|
81
|
+
# network round trip latency. I wouldn't recommend pushing more than
|
82
|
+
# 1000 per call but YMMV based on network quality, size of job args, etc.
|
83
|
+
# A large number of jobs can cause a bit of Redis command processing latency.
|
84
|
+
#
|
85
|
+
# Takes the same arguments as #push except that args is expected to be
|
86
|
+
# an Array of Arrays. All other keys are duplicated for each job. Each job
|
87
|
+
# is run through the client middleware pipeline and each job gets its own Job ID
|
88
|
+
# as normal.
|
89
|
+
#
|
90
|
+
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
|
+
# than the number given if the middleware stopped processing for one or more jobs.
|
92
|
+
def push_bulk(items)
|
93
|
+
arg = items['args'].first
|
94
|
+
return [] unless arg # no jobs to push
|
95
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
|
96
|
+
|
97
|
+
normed = normalize_item(items)
|
98
|
+
payloads = items['args'].map do |args|
|
99
|
+
copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
|
100
|
+
result = process_single(items['class'], copy)
|
101
|
+
result ? result : nil
|
102
|
+
end.compact
|
103
|
+
|
104
|
+
raw_push(payloads) if !payloads.empty?
|
105
|
+
payloads.collect { |payload| payload['jid'] }
|
106
|
+
end
|
107
|
+
|
108
|
+
# Allows sharding of jobs across any number of Redis instances. All jobs
|
109
|
+
# defined within the block will use the given Redis connection pool.
|
110
|
+
#
|
111
|
+
# pool = ConnectionPool.new { Redis.new }
|
112
|
+
# Sidekiq::Client.via(pool) do
|
113
|
+
# SomeWorker.perform_async(1,2,3)
|
114
|
+
# SomeOtherWorker.perform_async(1,2,3)
|
115
|
+
# end
|
116
|
+
#
|
117
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
118
|
+
# thousands of jobs per second. I do not recommend sharding unless
|
119
|
+
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
120
|
+
def self.via(pool)
|
121
|
+
raise ArgumentError, "No pool given" if pool.nil?
|
122
|
+
current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
|
123
|
+
Thread.current[:sidekiq_via_pool] = pool
|
124
|
+
yield
|
125
|
+
ensure
|
126
|
+
Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
|
127
|
+
end
|
128
|
+
|
129
|
+
class << self
|
130
|
+
|
131
|
+
def push(item)
|
132
|
+
new.push(item)
|
133
|
+
end
|
134
|
+
|
135
|
+
def push_bulk(items)
|
136
|
+
new.push_bulk(items)
|
137
|
+
end
|
138
|
+
|
139
|
+
# Resque compatibility helpers. Note all helpers
|
140
|
+
# should go through Worker#client_push.
|
141
|
+
#
|
142
|
+
# Example usage:
|
143
|
+
# Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
|
144
|
+
#
|
145
|
+
# Messages are enqueued to the 'default' queue.
|
146
|
+
#
|
147
|
+
def enqueue(klass, *args)
|
148
|
+
klass.client_push('class' => klass, 'args' => args)
|
149
|
+
end
|
150
|
+
|
151
|
+
# Example usage:
|
152
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
153
|
+
#
|
154
|
+
def enqueue_to(queue, klass, *args)
|
155
|
+
klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
|
156
|
+
end
|
157
|
+
|
158
|
+
# Example usage:
|
159
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
160
|
+
#
|
161
|
+
def enqueue_to_in(queue, interval, klass, *args)
|
162
|
+
int = interval.to_f
|
163
|
+
now = Time.now.to_f
|
164
|
+
ts = (int < 1_000_000_000 ? now + int : int)
|
165
|
+
|
166
|
+
item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
|
167
|
+
item.delete('at') if ts <= now
|
168
|
+
|
169
|
+
klass.client_push(item)
|
170
|
+
end
|
171
|
+
|
172
|
+
# Example usage:
|
173
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
174
|
+
#
|
175
|
+
def enqueue_in(interval, klass, *args)
|
176
|
+
klass.perform_in(interval, *args)
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
private
|
181
|
+
|
182
|
+
def raw_push(payloads)
|
183
|
+
@redis_pool.with do |conn|
|
184
|
+
conn.multi do
|
185
|
+
atomic_push(conn, payloads)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
true
|
189
|
+
end
|
190
|
+
|
191
|
+
def atomic_push(conn, payloads)
|
192
|
+
if payloads.first['at']
|
193
|
+
conn.zadd('schedule', payloads.map do |hash|
|
194
|
+
at = hash.delete('at').to_s
|
195
|
+
[at, Sidekiq.dump_json(hash)]
|
196
|
+
end)
|
197
|
+
else
|
198
|
+
q = payloads.first['queue']
|
199
|
+
now = Time.now.to_f
|
200
|
+
to_push = payloads.map do |entry|
|
201
|
+
entry['enqueued_at'] = now
|
202
|
+
Sidekiq.dump_json(entry)
|
203
|
+
end
|
204
|
+
conn.sadd('queues', q)
|
205
|
+
conn.lpush("queue:#{q}", to_push)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def process_single(worker_class, item)
|
210
|
+
queue = item['queue']
|
211
|
+
|
212
|
+
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
213
|
+
item
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def normalize_item(item)
|
218
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
|
219
|
+
raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
|
220
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
|
221
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
|
222
|
+
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
223
|
+
|
224
|
+
normalized_hash(item['class'])
|
225
|
+
.each{ |key, value| item[key] = value if item[key].nil? }
|
226
|
+
|
227
|
+
item['class'] = item['class'].to_s
|
228
|
+
item['queue'] = item['queue'].to_s
|
229
|
+
item['jid'] ||= SecureRandom.hex(12)
|
230
|
+
item['created_at'] ||= Time.now.to_f
|
231
|
+
item
|
232
|
+
end
|
233
|
+
|
234
|
+
def normalized_hash(item_class)
|
235
|
+
if item_class.is_a?(Class)
|
236
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
|
237
|
+
item_class.get_sidekiq_options
|
238
|
+
else
|
239
|
+
Sidekiq.default_worker_options
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
raise "no longer used, will be removed in 5.1"
|