sidekiq 4.2.10 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +573 -1
- data/LICENSE +3 -3
- data/README.md +25 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +81 -74
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +585 -285
- data/lib/sidekiq/cli.rb +256 -233
- data/lib/sidekiq/client.rb +86 -83
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +43 -0
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +13 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +282 -0
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +184 -90
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +43 -45
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +176 -91
- data/lib/sidekiq/rails.rb +41 -96
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +117 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +134 -44
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +80 -61
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +129 -86
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +170 -83
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +69 -109
- data/lib/sidekiq/worker.rb +290 -41
- data/lib/sidekiq.rb +185 -77
- data/sidekiq.gemspec +23 -27
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +70 -91
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +364 -144
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +15 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +19 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +5 -19
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +62 -24
- data/web/views/dashboard.erb +24 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +26 -12
- data/web/views/queues.erb +12 -2
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +66 -206
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -9
- data/.gitignore +0 -12
- data/.travis.yml +0 -18
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -173
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -628
- data/Rakefile +0 -12
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -119
- data/lib/sidekiq/exception_handler.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -31
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/lib/sidekiq/util.rb +0 -63
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,266 +1,290 @@
|
|
1
|
-
# encoding: utf-8
|
2
1
|
# frozen_string_literal: true
|
2
|
+
|
3
3
|
$stdout.sync = true
|
4
4
|
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
10
10
|
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
13
|
+
require "sidekiq/launcher"
|
13
14
|
|
14
|
-
module Sidekiq
|
15
|
+
module Sidekiq # :nodoc:
|
15
16
|
class CLI
|
16
|
-
include
|
17
|
+
include Sidekiq::Component
|
17
18
|
include Singleton unless $TESTING
|
18
19
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq'.freeze },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
|
-
# Used for CLI testing
|
28
|
-
attr_accessor :code
|
29
20
|
attr_accessor :launcher
|
30
21
|
attr_accessor :environment
|
22
|
+
attr_accessor :config
|
31
23
|
|
32
|
-
def
|
33
|
-
@
|
34
|
-
|
35
|
-
|
36
|
-
def parse(args=ARGV)
|
37
|
-
@code = nil
|
24
|
+
def parse(args = ARGV.dup)
|
25
|
+
@config = Sidekiq
|
26
|
+
@config[:error_handlers].clear
|
27
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
38
28
|
|
39
29
|
setup_options(args)
|
40
30
|
initialize_logger
|
41
31
|
validate!
|
42
|
-
|
43
|
-
|
32
|
+
end
|
33
|
+
|
34
|
+
def jruby?
|
35
|
+
defined?(::JRUBY_VERSION)
|
44
36
|
end
|
45
37
|
|
46
38
|
# Code within this method is not tested because it alters
|
47
39
|
# global process state irreversibly. PRs which improve the
|
48
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
49
|
-
def run
|
50
|
-
|
51
|
-
print_banner
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
52
43
|
|
53
|
-
|
44
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
|
+
print_banner
|
46
|
+
end
|
47
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
54
48
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
53
|
+
sigs.each do |sig|
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
59
62
|
end
|
60
|
-
|
61
|
-
puts "Signal #{sig} not supported"
|
63
|
+
self_write.puts(sig)
|
62
64
|
end
|
65
|
+
rescue ArgumentError
|
66
|
+
puts "Signal #{sig} not supported"
|
63
67
|
end
|
64
68
|
|
65
69
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
66
70
|
logger.info Sidekiq::LICENSE
|
67
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
71
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
68
72
|
|
69
73
|
# touch the connection pool so it is created before we
|
70
74
|
# fire startup and start multithreading.
|
71
|
-
|
72
|
-
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = info["redis_version"]
|
77
|
+
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
78
|
+
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction"
|
81
|
+
logger.warn <<~EOM
|
82
|
+
|
83
|
+
|
84
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
85
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
86
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
87
|
+
|
88
|
+
EOM
|
89
|
+
end
|
90
|
+
|
91
|
+
# Since the user can pass us a connection pool explicitly in the initializer, we
|
92
|
+
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
93
|
+
cursize = @config.redis_pool.size
|
94
|
+
needed = @config[:concurrency] + 2
|
95
|
+
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
96
|
+
|
97
|
+
# cache process identity
|
98
|
+
@config[:identity] = identity
|
73
99
|
|
74
100
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
75
|
-
|
101
|
+
@config.server_middleware
|
76
102
|
|
77
103
|
# Before this point, the process is initializing with just the main thread.
|
78
104
|
# Starting here the process will now have multiple threads running.
|
79
|
-
fire_event(:startup)
|
105
|
+
fire_event(:startup, reverse: false, reraise: true)
|
80
106
|
|
81
|
-
logger.debug { "Client Middleware: #{
|
82
|
-
logger.debug { "Server Middleware: #{
|
107
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
83
109
|
|
84
|
-
|
85
|
-
|
110
|
+
launch(self_read)
|
111
|
+
end
|
112
|
+
|
113
|
+
def launch(self_read)
|
114
|
+
if environment == "development" && $stdout.tty?
|
115
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
86
116
|
end
|
87
117
|
|
88
|
-
|
89
|
-
@launcher = Sidekiq::Launcher.new(options)
|
118
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
90
119
|
|
91
120
|
begin
|
92
121
|
launcher.run
|
93
122
|
|
94
|
-
while
|
95
|
-
signal =
|
123
|
+
while self_read.wait_readable
|
124
|
+
signal = self_read.gets.strip
|
96
125
|
handle_signal(signal)
|
97
126
|
end
|
98
127
|
rescue Interrupt
|
99
|
-
logger.info
|
128
|
+
logger.info "Shutting down"
|
100
129
|
launcher.stop
|
101
|
-
# Explicitly exit so busy Processor threads can't block
|
102
|
-
# process shutdown.
|
103
130
|
logger.info "Bye!"
|
131
|
+
|
132
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
133
|
+
#
|
134
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
135
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
136
|
+
# use the TTIN signal to determine where things are stuck.
|
104
137
|
exit(0)
|
105
138
|
end
|
106
139
|
end
|
107
140
|
|
141
|
+
def self.w
|
142
|
+
"\e[37m"
|
143
|
+
end
|
144
|
+
|
145
|
+
def self.r
|
146
|
+
"\e[31m"
|
147
|
+
end
|
148
|
+
|
149
|
+
def self.b
|
150
|
+
"\e[30m"
|
151
|
+
end
|
152
|
+
|
153
|
+
def self.reset
|
154
|
+
"\e[0m"
|
155
|
+
end
|
156
|
+
|
108
157
|
def self.banner
|
109
|
-
%
|
110
|
-
m,
|
111
|
-
`$b
|
112
|
-
.ss, $$: .,d$
|
113
|
-
`$$P,d$P' .,md$P"'
|
114
|
-
,$$$$$
|
115
|
-
.d
|
116
|
-
$$^' `"
|
117
|
-
$: ,$$:
|
118
|
-
`b :$$
|
119
|
-
$$:
|
120
|
-
$$ |____/|_
|
121
|
-
.d$$
|
122
|
-
}
|
158
|
+
%{
|
159
|
+
#{w} m,
|
160
|
+
#{w} `$b
|
161
|
+
#{w} .ss, $$: .,d$
|
162
|
+
#{w} `$$P,d$P' .,md$P"'
|
163
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
164
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
165
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
166
|
+
#{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
167
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
168
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
169
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
170
|
+
#{w} .d$$ #{r} |_|
|
171
|
+
#{reset}}
|
123
172
|
end
|
124
173
|
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
137
|
-
launcher.quiet
|
138
|
-
when 'TSTP'
|
139
|
-
# USR1 is not available on JVM, allow TSTP as an alternate signal
|
140
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
141
|
-
launcher.quiet
|
142
|
-
when 'USR2'
|
143
|
-
if Sidekiq.options[:logfile]
|
144
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
145
|
-
Sidekiq::Logging.reopen_logs
|
146
|
-
end
|
147
|
-
when 'TTIN'
|
174
|
+
SIGNAL_HANDLERS = {
|
175
|
+
# Ctrl-C in terminal
|
176
|
+
"INT" => ->(cli) { raise Interrupt },
|
177
|
+
# TERM is the signal that Sidekiq must exit.
|
178
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
179
|
+
"TERM" => ->(cli) { raise Interrupt },
|
180
|
+
"TSTP" => ->(cli) {
|
181
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
182
|
+
cli.launcher.quiet
|
183
|
+
},
|
184
|
+
"TTIN" => ->(cli) {
|
148
185
|
Thread.list.each do |thread|
|
149
|
-
|
186
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
150
187
|
if thread.backtrace
|
151
|
-
|
188
|
+
cli.logger.warn thread.backtrace.join("\n")
|
152
189
|
else
|
153
|
-
|
190
|
+
cli.logger.warn "<no backtrace available>"
|
154
191
|
end
|
155
192
|
end
|
156
|
-
|
193
|
+
}
|
194
|
+
}
|
195
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
196
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
197
|
+
|
198
|
+
def handle_signal(sig)
|
199
|
+
logger.debug "Got #{sig} signal"
|
200
|
+
SIGNAL_HANDLERS[sig].call(self)
|
157
201
|
end
|
158
202
|
|
159
203
|
private
|
160
204
|
|
161
205
|
def print_banner
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
puts Sidekiq::CLI.banner
|
166
|
-
puts "\e[0m"
|
167
|
-
end
|
206
|
+
puts "\e[31m"
|
207
|
+
puts Sidekiq::CLI.banner
|
208
|
+
puts "\e[0m"
|
168
209
|
end
|
169
210
|
|
170
|
-
def
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
files_to_reopen << file unless file.closed?
|
177
|
-
end
|
178
|
-
|
179
|
-
::Process.daemon(true, true)
|
180
|
-
|
181
|
-
files_to_reopen.each do |file|
|
182
|
-
begin
|
183
|
-
file.reopen file.path, "a+"
|
184
|
-
file.sync = true
|
185
|
-
rescue ::Exception
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
[$stdout, $stderr].each do |io|
|
190
|
-
File.open(options[:logfile], 'ab') do |f|
|
191
|
-
io.reopen(f)
|
192
|
-
end
|
193
|
-
io.sync = true
|
194
|
-
end
|
195
|
-
$stdin.reopen('/dev/null')
|
196
|
-
|
197
|
-
initialize_logger
|
211
|
+
def set_environment(cli_env)
|
212
|
+
# See #984 for discussion.
|
213
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
214
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
215
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
216
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
198
217
|
end
|
199
218
|
|
200
|
-
def
|
201
|
-
|
219
|
+
def symbolize_keys_deep!(hash)
|
220
|
+
hash.keys.each do |k|
|
221
|
+
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
222
|
+
hash[symkey] = hash.delete k
|
223
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
224
|
+
end
|
202
225
|
end
|
203
226
|
|
204
227
|
alias_method :die, :exit
|
205
228
|
alias_method :☠, :exit
|
206
229
|
|
207
230
|
def setup_options(args)
|
231
|
+
# parse CLI options
|
208
232
|
opts = parse_options(args)
|
233
|
+
|
209
234
|
set_environment opts[:environment]
|
210
235
|
|
211
|
-
|
212
|
-
|
236
|
+
# check config file presence
|
237
|
+
if opts[:config_file]
|
238
|
+
unless File.exist?(opts[:config_file])
|
239
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
240
|
+
end
|
241
|
+
else
|
242
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
243
|
+
File.join(opts[:require], "config")
|
244
|
+
else
|
245
|
+
File.join(@config[:require], "config")
|
246
|
+
end
|
213
247
|
|
214
|
-
|
215
|
-
|
216
|
-
|
248
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
249
|
+
path = File.join(config_dir, config_file)
|
250
|
+
opts[:config_file] ||= path if File.exist?(path)
|
251
|
+
end
|
252
|
+
end
|
217
253
|
|
218
|
-
options
|
219
|
-
|
254
|
+
# parse config file options
|
255
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
256
|
+
|
257
|
+
# set defaults
|
258
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
259
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
220
260
|
|
221
|
-
|
222
|
-
|
261
|
+
# merge with defaults
|
262
|
+
@config.merge!(opts)
|
223
263
|
end
|
224
264
|
|
225
|
-
def
|
226
|
-
ENV[
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
if ::Rails::VERSION::MAJOR < 4
|
233
|
-
require 'sidekiq/rails'
|
234
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
235
|
-
::Rails.application.eager_load!
|
236
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
237
|
-
# Painful contortions, see 1791 for discussion
|
238
|
-
# No autoloading, we want to force eager load for everything.
|
239
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
240
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
241
|
-
::Rails.application.config.eager_load = true
|
242
|
-
end
|
243
|
-
require 'sidekiq/rails'
|
244
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
265
|
+
def boot_application
|
266
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
267
|
+
|
268
|
+
if File.directory?(@config[:require])
|
269
|
+
require "rails"
|
270
|
+
if ::Rails::VERSION::MAJOR < 5
|
271
|
+
raise "Sidekiq no longer supports this version of Rails"
|
245
272
|
else
|
246
|
-
|
247
|
-
require
|
248
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
273
|
+
require "sidekiq/rails"
|
274
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
249
275
|
end
|
250
|
-
|
276
|
+
@config[:tag] ||= default_tag
|
251
277
|
else
|
252
|
-
|
253
|
-
"./#{options[:require]} or /path/to/#{options[:require]}"
|
254
|
-
|
255
|
-
require(options[:require]) || raise(ArgumentError, not_required_message)
|
278
|
+
require @config[:require]
|
256
279
|
end
|
257
280
|
end
|
258
281
|
|
259
282
|
def default_tag
|
260
283
|
dir = ::Rails.root
|
261
284
|
name = File.basename(dir)
|
262
|
-
|
263
|
-
|
285
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
286
|
+
if name.to_i != 0 && prevdir
|
287
|
+
if File.basename(prevdir) == "releases"
|
264
288
|
return File.basename(File.dirname(prevdir))
|
265
289
|
end
|
266
290
|
end
|
@@ -268,57 +292,56 @@ module Sidekiq
|
|
268
292
|
end
|
269
293
|
|
270
294
|
def validate!
|
271
|
-
|
272
|
-
|
273
|
-
if !File.exist?(options[:require]) ||
|
274
|
-
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
295
|
+
if !File.exist?(@config[:require]) ||
|
296
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
275
297
|
logger.info "=================================================================="
|
276
|
-
logger.info " Please point
|
277
|
-
logger.info " to load your
|
298
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
299
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
278
300
|
logger.info "=================================================================="
|
279
301
|
logger.info @parser
|
280
302
|
die(1)
|
281
303
|
end
|
282
304
|
|
283
305
|
[:concurrency, :timeout].each do |opt|
|
284
|
-
raise ArgumentError, "#{opt}: #{
|
306
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
285
307
|
end
|
286
308
|
end
|
287
309
|
|
288
310
|
def parse_options(argv)
|
289
311
|
opts = {}
|
312
|
+
@parser = option_parser(opts)
|
313
|
+
@parser.parse!(argv)
|
314
|
+
opts
|
315
|
+
end
|
290
316
|
|
291
|
-
|
292
|
-
|
317
|
+
def option_parser(opts)
|
318
|
+
parser = OptionParser.new { |o|
|
319
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
293
320
|
opts[:concurrency] = Integer(arg)
|
294
321
|
end
|
295
322
|
|
296
|
-
o.on
|
297
|
-
|
323
|
+
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
324
|
+
puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
298
325
|
end
|
299
326
|
|
300
|
-
o.on
|
327
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
301
328
|
opts[:environment] = arg
|
302
329
|
end
|
303
330
|
|
304
|
-
o.on
|
331
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
305
332
|
opts[:tag] = arg
|
306
333
|
end
|
307
334
|
|
308
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
309
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
310
|
-
end
|
311
|
-
|
312
335
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
313
336
|
queue, weight = arg.split(",")
|
314
337
|
parse_queue opts, queue, weight
|
315
338
|
end
|
316
339
|
|
317
|
-
o.on
|
340
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
318
341
|
opts[:require] = arg
|
319
342
|
end
|
320
343
|
|
321
|
-
o.on
|
344
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
322
345
|
opts[:timeout] = Integer(arg)
|
323
346
|
end
|
324
347
|
|
@@ -326,81 +349,81 @@ module Sidekiq
|
|
326
349
|
opts[:verbose] = arg
|
327
350
|
end
|
328
351
|
|
329
|
-
o.on
|
352
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
330
353
|
opts[:config_file] = arg
|
331
354
|
end
|
332
355
|
|
333
|
-
o.on
|
334
|
-
|
356
|
+
o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
|
357
|
+
puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
335
358
|
end
|
336
359
|
|
337
|
-
o.on
|
338
|
-
|
360
|
+
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
361
|
+
puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
339
362
|
end
|
340
363
|
|
341
|
-
o.on
|
364
|
+
o.on "-V", "--version", "Print version and exit" do |arg|
|
342
365
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
343
366
|
die(0)
|
344
367
|
end
|
345
|
-
|
368
|
+
}
|
346
369
|
|
347
|
-
|
348
|
-
|
349
|
-
logger.info
|
370
|
+
parser.banner = "sidekiq [options]"
|
371
|
+
parser.on_tail "-h", "--help", "Show help" do
|
372
|
+
logger.info parser
|
350
373
|
die 1
|
351
374
|
end
|
352
|
-
@parser.parse!(argv)
|
353
|
-
|
354
|
-
%w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
|
355
|
-
opts[:config_file] ||= filename if File.exist?(filename)
|
356
|
-
end
|
357
375
|
|
358
|
-
|
376
|
+
parser
|
359
377
|
end
|
360
378
|
|
361
379
|
def initialize_logger
|
362
|
-
|
363
|
-
|
364
|
-
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
380
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
365
381
|
end
|
366
382
|
|
367
|
-
def
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
383
|
+
def parse_config(path)
|
384
|
+
erb = ERB.new(File.read(path))
|
385
|
+
erb.filename = File.expand_path(path)
|
386
|
+
opts = load_yaml(erb.result) || {}
|
387
|
+
|
388
|
+
if opts.respond_to? :deep_symbolize_keys!
|
389
|
+
opts.deep_symbolize_keys!
|
390
|
+
else
|
391
|
+
symbolize_keys_deep!(opts)
|
373
392
|
end
|
393
|
+
|
394
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
395
|
+
opts.delete(:strict)
|
396
|
+
|
397
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
398
|
+
|
399
|
+
opts
|
374
400
|
end
|
375
401
|
|
376
|
-
def
|
377
|
-
|
378
|
-
|
379
|
-
opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
|
380
|
-
opts = opts.merge(opts.delete(environment) || {})
|
381
|
-
parse_queues(opts, opts.delete(:queues) || [])
|
402
|
+
def load_yaml(src)
|
403
|
+
if Psych::VERSION > "4.0"
|
404
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
382
405
|
else
|
383
|
-
|
384
|
-
# can be deployed by cap with just the defaults.
|
385
|
-
end
|
386
|
-
ns = opts.delete(:namespace)
|
387
|
-
if ns
|
388
|
-
# logger hasn't been initialized yet, puts is all we have.
|
389
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
390
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
406
|
+
YAML.load(src)
|
391
407
|
end
|
392
|
-
opts
|
393
408
|
end
|
394
409
|
|
395
410
|
def parse_queues(opts, queues_and_weights)
|
396
411
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
397
412
|
end
|
398
413
|
|
399
|
-
def parse_queue(opts,
|
400
|
-
[
|
401
|
-
|
402
|
-
|
414
|
+
def parse_queue(opts, queue, weight = nil)
|
415
|
+
opts[:queues] ||= []
|
416
|
+
opts[:strict] = true if opts[:strict].nil?
|
417
|
+
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
418
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
403
419
|
opts[:strict] = false if weight.to_i > 0
|
404
420
|
end
|
421
|
+
|
422
|
+
def rails_app?
|
423
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
424
|
+
end
|
405
425
|
end
|
406
426
|
end
|
427
|
+
|
428
|
+
require "sidekiq/systemd"
|
429
|
+
require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
|