sidekiq 5.1.1 → 6.5.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +507 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +80 -68
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +537 -286
- data/lib/sidekiq/cli.rb +243 -240
- data/lib/sidekiq/client.rb +82 -85
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +9 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +36 -9
- data/lib/sidekiq/job_retry.rb +143 -97
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +185 -85
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +41 -43
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +156 -98
- data/lib/sidekiq/rails.rb +48 -42
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +109 -51
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +133 -41
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +72 -62
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +127 -76
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +133 -96
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +69 -109
- data/lib/sidekiq/worker.rb +268 -102
- data/lib/sidekiq.rb +175 -66
- data/sidekiq.gemspec +23 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +65 -91
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +120 -232
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +9 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +14 -1
- data/web/locales/es.yml +21 -5
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +14 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +4 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +61 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +4 -2
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +24 -10
- data/web/views/queues.erb +11 -3
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +62 -135
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -210
- data/Gemfile +0 -8
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -716
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -22
- data/lib/sidekiq/middleware/server/active_record_cache.rb +0 -11
- data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,46 +1,45 @@
|
|
1
|
-
# encoding: utf-8
|
2
1
|
# frozen_string_literal: true
|
3
|
-
$stdout.sync = true
|
4
|
-
|
5
|
-
require 'yaml'
|
6
|
-
require 'singleton'
|
7
|
-
require 'optparse'
|
8
|
-
require 'erb'
|
9
|
-
require 'fileutils'
|
10
2
|
|
11
|
-
|
12
|
-
require 'sidekiq/util'
|
3
|
+
$stdout.sync = true
|
13
4
|
|
14
|
-
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
10
|
+
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
13
|
+
require "sidekiq/launcher"
|
14
|
+
|
15
|
+
# module ScoutApm
|
16
|
+
# VERSION = "5.3.1"
|
17
|
+
# end
|
18
|
+
fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
|
19
|
+
|
20
|
+
|
21
|
+
scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
|
22
|
+
|
23
|
+
|
24
|
+
EOM
|
25
|
+
|
26
|
+
module Sidekiq # :nodoc:
|
15
27
|
class CLI
|
16
|
-
include
|
28
|
+
include Sidekiq::Component
|
17
29
|
include Singleton unless $TESTING
|
18
30
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq'.freeze },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
|
-
# Used for CLI testing
|
28
|
-
attr_accessor :code
|
29
31
|
attr_accessor :launcher
|
30
32
|
attr_accessor :environment
|
33
|
+
attr_accessor :config
|
31
34
|
|
32
|
-
def
|
33
|
-
@
|
34
|
-
|
35
|
-
|
36
|
-
def parse(args=ARGV)
|
37
|
-
@code = nil
|
35
|
+
def parse(args = ARGV.dup)
|
36
|
+
@config = Sidekiq
|
37
|
+
@config[:error_handlers].clear
|
38
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
38
39
|
|
39
40
|
setup_options(args)
|
40
41
|
initialize_logger
|
41
42
|
validate!
|
42
|
-
daemonize
|
43
|
-
write_pid
|
44
43
|
end
|
45
44
|
|
46
45
|
def jruby?
|
@@ -50,183 +49,190 @@ module Sidekiq
|
|
50
49
|
# Code within this method is not tested because it alters
|
51
50
|
# global process state irreversibly. PRs which improve the
|
52
51
|
# test coverage of Sidekiq::CLI are welcomed.
|
53
|
-
def run
|
54
|
-
|
55
|
-
print_banner
|
52
|
+
def run(boot_app: true)
|
53
|
+
boot_application if boot_app
|
56
54
|
|
57
|
-
|
58
|
-
|
59
|
-
# USR1 and USR2 don't work on the JVM
|
60
|
-
if !jruby?
|
61
|
-
sigs << 'USR1'
|
62
|
-
sigs << 'USR2'
|
55
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
56
|
+
print_banner
|
63
57
|
end
|
58
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
64
59
|
|
60
|
+
self_read, self_write = IO.pipe
|
61
|
+
sigs = %w[INT TERM TTIN TSTP]
|
62
|
+
# USR1 and USR2 don't work on the JVM
|
63
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
65
64
|
sigs.each do |sig|
|
66
|
-
|
67
|
-
|
68
|
-
|
65
|
+
old_handler = Signal.trap(sig) do
|
66
|
+
if old_handler.respond_to?(:call)
|
67
|
+
begin
|
68
|
+
old_handler.call
|
69
|
+
rescue Exception => exc
|
70
|
+
# signal handlers can't use Logger so puts only
|
71
|
+
puts ["Error in #{sig} handler", exc].inspect
|
72
|
+
end
|
69
73
|
end
|
70
|
-
|
71
|
-
puts "Signal #{sig} not supported"
|
74
|
+
self_write.puts(sig)
|
72
75
|
end
|
76
|
+
rescue ArgumentError
|
77
|
+
puts "Signal #{sig} not supported"
|
73
78
|
end
|
74
79
|
|
75
80
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
76
81
|
logger.info Sidekiq::LICENSE
|
77
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
82
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
78
83
|
|
79
84
|
# touch the connection pool so it is created before we
|
80
85
|
# fire startup and start multithreading.
|
81
|
-
|
82
|
-
|
86
|
+
info = @config.redis_info
|
87
|
+
ver = info["redis_version"]
|
88
|
+
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
89
|
+
|
90
|
+
maxmemory_policy = info["maxmemory_policy"]
|
91
|
+
if maxmemory_policy != "noeviction"
|
92
|
+
logger.warn <<~EOM
|
93
|
+
|
94
|
+
|
95
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
96
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
97
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
98
|
+
|
99
|
+
EOM
|
100
|
+
end
|
101
|
+
|
102
|
+
# Since the user can pass us a connection pool explicitly in the initializer, we
|
103
|
+
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
104
|
+
cursize = @config.redis_pool.size
|
105
|
+
needed = @config[:concurrency] + 2
|
106
|
+
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
83
107
|
|
84
108
|
# cache process identity
|
85
|
-
|
109
|
+
@config[:identity] = identity
|
86
110
|
|
87
111
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
88
|
-
|
112
|
+
@config.server_middleware
|
89
113
|
|
90
114
|
# Before this point, the process is initializing with just the main thread.
|
91
115
|
# Starting here the process will now have multiple threads running.
|
92
116
|
fire_event(:startup, reverse: false, reraise: true)
|
93
117
|
|
94
|
-
logger.debug { "Client Middleware: #{
|
95
|
-
logger.debug { "Server Middleware: #{
|
118
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
119
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
120
|
+
|
121
|
+
launch(self_read)
|
122
|
+
end
|
96
123
|
|
97
|
-
|
98
|
-
|
124
|
+
def launch(self_read)
|
125
|
+
if environment == "development" && $stdout.tty?
|
126
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
99
127
|
end
|
100
128
|
|
101
|
-
|
102
|
-
@launcher = Sidekiq::Launcher.new(options)
|
129
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
103
130
|
|
104
131
|
begin
|
105
132
|
launcher.run
|
106
133
|
|
107
|
-
while
|
108
|
-
signal =
|
134
|
+
while self_read.wait_readable
|
135
|
+
signal = self_read.gets.strip
|
109
136
|
handle_signal(signal)
|
110
137
|
end
|
111
138
|
rescue Interrupt
|
112
|
-
logger.info
|
139
|
+
logger.info "Shutting down"
|
113
140
|
launcher.stop
|
114
|
-
# Explicitly exit so busy Processor threads can't block
|
115
|
-
# process shutdown.
|
116
141
|
logger.info "Bye!"
|
142
|
+
|
143
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
144
|
+
#
|
145
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
146
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
147
|
+
# use the TTIN signal to determine where things are stuck.
|
117
148
|
exit(0)
|
118
149
|
end
|
119
150
|
end
|
120
151
|
|
152
|
+
def self.w
|
153
|
+
"\e[37m"
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.r
|
157
|
+
"\e[31m"
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.b
|
161
|
+
"\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.reset
|
165
|
+
"\e[0m"
|
166
|
+
end
|
167
|
+
|
121
168
|
def self.banner
|
122
|
-
%
|
123
|
-
m,
|
124
|
-
`$b
|
125
|
-
.ss, $$: .,d$
|
126
|
-
`$$P,d$P' .,md$P"'
|
127
|
-
,$$$$$
|
128
|
-
.d
|
129
|
-
$$^' `"
|
130
|
-
$: ,$$:
|
131
|
-
`b :$$
|
132
|
-
$$:
|
133
|
-
$$ |____/|_
|
134
|
-
.d$$
|
135
|
-
}
|
169
|
+
%{
|
170
|
+
#{w} m,
|
171
|
+
#{w} `$b
|
172
|
+
#{w} .ss, $$: .,d$
|
173
|
+
#{w} `$$P,d$P' .,md$P"'
|
174
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
175
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
176
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
177
|
+
#{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
178
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
179
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
180
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
181
|
+
#{w} .d$$ #{r} |_|
|
182
|
+
#{reset}}
|
136
183
|
end
|
137
184
|
|
138
185
|
SIGNAL_HANDLERS = {
|
139
186
|
# Ctrl-C in terminal
|
140
|
-
|
187
|
+
"INT" => ->(cli) { raise Interrupt },
|
141
188
|
# TERM is the signal that Sidekiq must exit.
|
142
189
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
cli.launcher.quiet
|
147
|
-
},
|
148
|
-
'TSTP' => ->(cli) {
|
149
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
190
|
+
"TERM" => ->(cli) { raise Interrupt },
|
191
|
+
"TSTP" => ->(cli) {
|
192
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
150
193
|
cli.launcher.quiet
|
151
194
|
},
|
152
|
-
|
153
|
-
if Sidekiq.options[:logfile]
|
154
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
155
|
-
Sidekiq::Logging.reopen_logs
|
156
|
-
end
|
157
|
-
},
|
158
|
-
'TTIN' => ->(cli) {
|
195
|
+
"TTIN" => ->(cli) {
|
159
196
|
Thread.list.each do |thread|
|
160
|
-
|
197
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
161
198
|
if thread.backtrace
|
162
|
-
|
199
|
+
cli.logger.warn thread.backtrace.join("\n")
|
163
200
|
else
|
164
|
-
|
201
|
+
cli.logger.warn "<no backtrace available>"
|
165
202
|
end
|
166
203
|
end
|
167
|
-
}
|
204
|
+
}
|
168
205
|
}
|
206
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
207
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
169
208
|
|
170
209
|
def handle_signal(sig)
|
171
|
-
|
172
|
-
|
173
|
-
if handy
|
174
|
-
handy.call(self)
|
175
|
-
else
|
176
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
177
|
-
end
|
210
|
+
logger.debug "Got #{sig} signal"
|
211
|
+
SIGNAL_HANDLERS[sig].call(self)
|
178
212
|
end
|
179
213
|
|
180
214
|
private
|
181
215
|
|
182
216
|
def print_banner
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
puts Sidekiq::CLI.banner
|
187
|
-
puts "\e[0m"
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
def daemonize
|
192
|
-
return unless options[:daemon]
|
193
|
-
|
194
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
195
|
-
files_to_reopen = []
|
196
|
-
ObjectSpace.each_object(File) do |file|
|
197
|
-
files_to_reopen << file unless file.closed?
|
198
|
-
end
|
199
|
-
|
200
|
-
::Process.daemon(true, true)
|
201
|
-
|
202
|
-
files_to_reopen.each do |file|
|
203
|
-
begin
|
204
|
-
file.reopen file.path, "a+"
|
205
|
-
file.sync = true
|
206
|
-
rescue ::Exception
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
|
-
[$stdout, $stderr].each do |io|
|
211
|
-
File.open(options[:logfile], 'ab') do |f|
|
212
|
-
io.reopen(f)
|
213
|
-
end
|
214
|
-
io.sync = true
|
215
|
-
end
|
216
|
-
$stdin.reopen('/dev/null')
|
217
|
-
|
218
|
-
initialize_logger
|
217
|
+
puts "\e[31m"
|
218
|
+
puts Sidekiq::CLI.banner
|
219
|
+
puts "\e[0m"
|
219
220
|
end
|
220
221
|
|
221
222
|
def set_environment(cli_env)
|
222
|
-
|
223
|
+
# See #984 for discussion.
|
224
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
225
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
226
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
227
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
228
|
+
config[:environment] = @environment
|
223
229
|
end
|
224
230
|
|
225
231
|
def symbolize_keys_deep!(hash)
|
226
232
|
hash.keys.each do |k|
|
227
233
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
228
234
|
hash[symkey] = hash.delete k
|
229
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
235
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
230
236
|
end
|
231
237
|
end
|
232
238
|
|
@@ -234,58 +240,63 @@ module Sidekiq
|
|
234
240
|
alias_method :☠, :exit
|
235
241
|
|
236
242
|
def setup_options(args)
|
243
|
+
# parse CLI options
|
237
244
|
opts = parse_options(args)
|
245
|
+
|
238
246
|
set_environment opts[:environment]
|
239
247
|
|
240
|
-
|
241
|
-
|
248
|
+
# check config file presence
|
249
|
+
if opts[:config_file]
|
250
|
+
unless File.exist?(opts[:config_file])
|
251
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
252
|
+
end
|
253
|
+
else
|
254
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
255
|
+
File.join(opts[:require], "config")
|
256
|
+
else
|
257
|
+
File.join(@config[:require], "config")
|
258
|
+
end
|
242
259
|
|
243
|
-
|
244
|
-
|
260
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
261
|
+
path = File.join(config_dir, config_file)
|
262
|
+
opts[:config_file] ||= path if File.exist?(path)
|
263
|
+
end
|
264
|
+
end
|
245
265
|
|
246
|
-
options
|
247
|
-
|
266
|
+
# parse config file options
|
267
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
248
268
|
|
249
|
-
|
250
|
-
|
251
|
-
|
269
|
+
# set defaults
|
270
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
271
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
252
272
|
|
253
|
-
|
254
|
-
|
273
|
+
# merge with defaults
|
274
|
+
@config.merge!(opts)
|
275
|
+
end
|
255
276
|
|
256
|
-
|
277
|
+
def boot_application
|
278
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
257
279
|
|
258
|
-
if File.directory?(
|
259
|
-
require
|
260
|
-
if ::Rails::VERSION::MAJOR <
|
280
|
+
if File.directory?(@config[:require])
|
281
|
+
require "rails"
|
282
|
+
if ::Rails::VERSION::MAJOR < 5
|
261
283
|
raise "Sidekiq no longer supports this version of Rails"
|
262
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
263
|
-
# Painful contortions, see 1791 for discussion
|
264
|
-
# No autoloading, we want to force eager load for everything.
|
265
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
266
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
267
|
-
::Rails.application.config.eager_load = true
|
268
|
-
end
|
269
|
-
require 'sidekiq/rails'
|
270
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
271
284
|
else
|
272
|
-
require
|
273
|
-
require File.expand_path("#{
|
285
|
+
require "sidekiq/rails"
|
286
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
274
287
|
end
|
275
|
-
|
288
|
+
@config[:tag] ||= default_tag
|
276
289
|
else
|
277
|
-
|
278
|
-
"./#{options[:require]} or /path/to/#{options[:require]}"
|
279
|
-
|
280
|
-
require(options[:require]) || raise(ArgumentError, not_required_message)
|
290
|
+
require @config[:require]
|
281
291
|
end
|
282
292
|
end
|
283
293
|
|
284
294
|
def default_tag
|
285
295
|
dir = ::Rails.root
|
286
296
|
name = File.basename(dir)
|
287
|
-
|
288
|
-
|
297
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
298
|
+
if name.to_i != 0 && prevdir
|
299
|
+
if File.basename(prevdir) == "releases"
|
289
300
|
return File.basename(File.dirname(prevdir))
|
290
301
|
end
|
291
302
|
end
|
@@ -293,57 +304,56 @@ module Sidekiq
|
|
293
304
|
end
|
294
305
|
|
295
306
|
def validate!
|
296
|
-
|
297
|
-
|
298
|
-
if !File.exist?(options[:require]) ||
|
299
|
-
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
307
|
+
if !File.exist?(@config[:require]) ||
|
308
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
300
309
|
logger.info "=================================================================="
|
301
|
-
logger.info " Please point
|
302
|
-
logger.info " to load your
|
310
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
311
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
303
312
|
logger.info "=================================================================="
|
304
313
|
logger.info @parser
|
305
314
|
die(1)
|
306
315
|
end
|
307
316
|
|
308
317
|
[:concurrency, :timeout].each do |opt|
|
309
|
-
raise ArgumentError, "#{opt}: #{
|
318
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
310
319
|
end
|
311
320
|
end
|
312
321
|
|
313
322
|
def parse_options(argv)
|
314
323
|
opts = {}
|
324
|
+
@parser = option_parser(opts)
|
325
|
+
@parser.parse!(argv)
|
326
|
+
opts
|
327
|
+
end
|
315
328
|
|
316
|
-
|
317
|
-
|
329
|
+
def option_parser(opts)
|
330
|
+
parser = OptionParser.new { |o|
|
331
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
318
332
|
opts[:concurrency] = Integer(arg)
|
319
333
|
end
|
320
334
|
|
321
|
-
o.on
|
322
|
-
|
335
|
+
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
336
|
+
puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
323
337
|
end
|
324
338
|
|
325
|
-
o.on
|
339
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
326
340
|
opts[:environment] = arg
|
327
341
|
end
|
328
342
|
|
329
|
-
o.on
|
343
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
330
344
|
opts[:tag] = arg
|
331
345
|
end
|
332
346
|
|
333
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
334
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
335
|
-
end
|
336
|
-
|
337
347
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
338
348
|
queue, weight = arg.split(",")
|
339
349
|
parse_queue opts, queue, weight
|
340
350
|
end
|
341
351
|
|
342
|
-
o.on
|
352
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
343
353
|
opts[:require] = arg
|
344
354
|
end
|
345
355
|
|
346
|
-
o.on
|
356
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
347
357
|
opts[:timeout] = Integer(arg)
|
348
358
|
end
|
349
359
|
|
@@ -351,88 +361,81 @@ module Sidekiq
|
|
351
361
|
opts[:verbose] = arg
|
352
362
|
end
|
353
363
|
|
354
|
-
o.on
|
364
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
355
365
|
opts[:config_file] = arg
|
356
366
|
end
|
357
367
|
|
358
|
-
o.on
|
359
|
-
|
368
|
+
o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
|
369
|
+
puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
360
370
|
end
|
361
371
|
|
362
|
-
o.on
|
363
|
-
|
372
|
+
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
373
|
+
puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
364
374
|
end
|
365
375
|
|
366
|
-
o.on
|
376
|
+
o.on "-V", "--version", "Print version and exit" do |arg|
|
367
377
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
368
378
|
die(0)
|
369
379
|
end
|
370
|
-
|
380
|
+
}
|
371
381
|
|
372
|
-
|
373
|
-
|
374
|
-
logger.info
|
382
|
+
parser.banner = "sidekiq [options]"
|
383
|
+
parser.on_tail "-h", "--help", "Show help" do
|
384
|
+
logger.info parser
|
375
385
|
die 1
|
376
386
|
end
|
377
|
-
@parser.parse!(argv)
|
378
|
-
|
379
|
-
%w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
|
380
|
-
opts[:config_file] ||= filename if File.exist?(filename)
|
381
|
-
end
|
382
387
|
|
383
|
-
|
388
|
+
parser
|
384
389
|
end
|
385
390
|
|
386
391
|
def initialize_logger
|
387
|
-
|
388
|
-
|
389
|
-
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
392
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
390
393
|
end
|
391
394
|
|
392
|
-
def
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
395
|
+
def parse_config(path)
|
396
|
+
erb = ERB.new(File.read(path))
|
397
|
+
erb.filename = File.expand_path(path)
|
398
|
+
opts = load_yaml(erb.result) || {}
|
399
|
+
|
400
|
+
if opts.respond_to? :deep_symbolize_keys!
|
401
|
+
opts.deep_symbolize_keys!
|
402
|
+
else
|
403
|
+
symbolize_keys_deep!(opts)
|
398
404
|
end
|
399
|
-
end
|
400
405
|
|
401
|
-
|
402
|
-
opts
|
403
|
-
if File.exist?(cfile)
|
404
|
-
opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
|
406
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
407
|
+
opts.delete(:strict)
|
405
408
|
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
end
|
409
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
410
|
+
|
411
|
+
opts
|
412
|
+
end
|
411
413
|
|
412
|
-
|
413
|
-
|
414
|
+
def load_yaml(src)
|
415
|
+
if Psych::VERSION > "4.0"
|
416
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
414
417
|
else
|
415
|
-
|
416
|
-
# can be deployed by cap with just the defaults.
|
417
|
-
end
|
418
|
-
ns = opts.delete(:namespace)
|
419
|
-
if ns
|
420
|
-
# logger hasn't been initialized yet, puts is all we have.
|
421
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
422
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
418
|
+
YAML.load(src)
|
423
419
|
end
|
424
|
-
opts
|
425
420
|
end
|
426
421
|
|
427
422
|
def parse_queues(opts, queues_and_weights)
|
428
423
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
429
424
|
end
|
430
425
|
|
431
|
-
def parse_queue(opts,
|
432
|
-
[
|
433
|
-
|
434
|
-
|
426
|
+
def parse_queue(opts, queue, weight = nil)
|
427
|
+
opts[:queues] ||= []
|
428
|
+
opts[:strict] = true if opts[:strict].nil?
|
429
|
+
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
430
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
435
431
|
opts[:strict] = false if weight.to_i > 0
|
436
432
|
end
|
433
|
+
|
434
|
+
def rails_app?
|
435
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
436
|
+
end
|
437
437
|
end
|
438
438
|
end
|
439
|
+
|
440
|
+
require "sidekiq/systemd"
|
441
|
+
require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
|