sidekiq 5.1.1 → 7.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +627 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +566 -329
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +241 -256
- data/lib/sidekiq/client.rb +125 -102
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +278 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +36 -9
- data/lib/sidekiq/job_retry.rb +147 -98
- data/lib/sidekiq/job_util.rb +105 -0
- data/lib/sidekiq/launcher.rb +207 -103
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -47
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +159 -107
- data/lib/sidekiq/rails.rb +54 -43
- data/lib/sidekiq/redis_client_adapter.rb +96 -0
- data/lib/sidekiq/redis_connection.rb +39 -81
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +139 -48
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +70 -88
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +143 -77
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +144 -106
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +60 -111
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +94 -183
- data/sidekiq.gemspec +25 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +130 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +36 -282
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +134 -521
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +71 -64
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +84 -65
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +8 -2
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +75 -25
- data/web/views/dashboard.erb +58 -18
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +4 -2
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -23
- data/web/views/queues.erb +14 -4
- data/web/views/retries.erb +19 -16
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +71 -140
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -210
- data/Gemfile +0 -8
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -716
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/delay.rb +0 -41
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -22
- data/lib/sidekiq/middleware/server/active_record_cache.rb +0 -11
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -204
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,46 +1,34 @@
|
|
1
|
-
# encoding: utf-8
|
2
1
|
# frozen_string_literal: true
|
2
|
+
|
3
3
|
$stdout.sync = true
|
4
4
|
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
10
10
|
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
13
|
+
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
15
|
+
require "sidekiq/launcher"
|
13
16
|
|
14
|
-
module Sidekiq
|
17
|
+
module Sidekiq # :nodoc:
|
15
18
|
class CLI
|
16
|
-
include
|
19
|
+
include Sidekiq::Component
|
17
20
|
include Singleton unless $TESTING
|
18
21
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq'.freeze },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
|
-
# Used for CLI testing
|
28
|
-
attr_accessor :code
|
29
22
|
attr_accessor :launcher
|
30
23
|
attr_accessor :environment
|
24
|
+
attr_accessor :config
|
31
25
|
|
32
|
-
def
|
33
|
-
@
|
34
|
-
end
|
35
|
-
|
36
|
-
def parse(args=ARGV)
|
37
|
-
@code = nil
|
26
|
+
def parse(args = ARGV.dup)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
38
28
|
|
39
29
|
setup_options(args)
|
40
30
|
initialize_logger
|
41
31
|
validate!
|
42
|
-
daemonize
|
43
|
-
write_pid
|
44
32
|
end
|
45
33
|
|
46
34
|
def jruby?
|
@@ -50,183 +38,206 @@ module Sidekiq
|
|
50
38
|
# Code within this method is not tested because it alters
|
51
39
|
# global process state irreversibly. PRs which improve the
|
52
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
53
|
-
def run
|
54
|
-
|
55
|
-
print_banner
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
56
43
|
|
57
|
-
|
58
|
-
|
59
|
-
# USR1 and USR2 don't work on the JVM
|
60
|
-
if !jruby?
|
61
|
-
sigs << 'USR1'
|
62
|
-
sigs << 'USR2'
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
|
+
print_banner
|
63
46
|
end
|
47
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
64
48
|
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
65
53
|
sigs.each do |sig|
|
66
|
-
|
67
|
-
|
68
|
-
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
69
62
|
end
|
70
|
-
|
71
|
-
puts "Signal #{sig} not supported"
|
63
|
+
self_write.puts(sig)
|
72
64
|
end
|
65
|
+
rescue ArgumentError
|
66
|
+
puts "Signal #{sig} not supported"
|
73
67
|
end
|
74
68
|
|
75
69
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
76
70
|
logger.info Sidekiq::LICENSE
|
77
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
71
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
78
72
|
|
79
73
|
# touch the connection pool so it is created before we
|
80
74
|
# fire startup and start multithreading.
|
81
|
-
|
82
|
-
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
78
|
+
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
81
|
+
# Redis Enterprise Cloud returns "" for their policy 😳
|
82
|
+
logger.warn <<~EOM
|
83
|
+
|
84
|
+
|
85
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
86
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
87
|
+
See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
|
88
|
+
|
89
|
+
EOM
|
90
|
+
end
|
91
|
+
|
92
|
+
# Since the user can pass us a connection pool explicitly in the initializer, we
|
93
|
+
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
94
|
+
@config.capsules.each_pair do |name, cap|
|
95
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
96
|
+
end
|
83
97
|
|
84
98
|
# cache process identity
|
85
|
-
|
99
|
+
@config[:identity] = identity
|
86
100
|
|
87
101
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
88
|
-
|
102
|
+
@config.server_middleware
|
89
103
|
|
90
104
|
# Before this point, the process is initializing with just the main thread.
|
91
105
|
# Starting here the process will now have multiple threads running.
|
92
106
|
fire_event(:startup, reverse: false, reraise: true)
|
93
107
|
|
94
|
-
logger.debug { "Client Middleware: #{
|
95
|
-
logger.debug { "Server Middleware: #{
|
108
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
109
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
110
|
+
|
111
|
+
launch(self_read)
|
112
|
+
end
|
96
113
|
|
97
|
-
|
98
|
-
|
114
|
+
def launch(self_read)
|
115
|
+
if environment == "development" && $stdout.tty?
|
116
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
99
117
|
end
|
100
118
|
|
101
|
-
|
102
|
-
@launcher = Sidekiq::Launcher.new(options)
|
119
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
103
120
|
|
104
121
|
begin
|
105
122
|
launcher.run
|
106
123
|
|
107
|
-
while
|
108
|
-
signal =
|
124
|
+
while self_read.wait_readable
|
125
|
+
signal = self_read.gets.strip
|
109
126
|
handle_signal(signal)
|
110
127
|
end
|
111
128
|
rescue Interrupt
|
112
|
-
logger.info
|
129
|
+
logger.info "Shutting down"
|
113
130
|
launcher.stop
|
114
|
-
# Explicitly exit so busy Processor threads can't block
|
115
|
-
# process shutdown.
|
116
131
|
logger.info "Bye!"
|
132
|
+
|
133
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
134
|
+
#
|
135
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
136
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
137
|
+
# use the TTIN signal to determine where things are stuck.
|
117
138
|
exit(0)
|
118
139
|
end
|
119
140
|
end
|
120
141
|
|
142
|
+
HOLIDAY_COLORS = {
|
143
|
+
# got other color-specific holidays from around the world?
|
144
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
145
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
146
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
147
|
+
}
|
148
|
+
|
149
|
+
def self.day
|
150
|
+
@@day ||= begin
|
151
|
+
t = Date.today
|
152
|
+
"#{t.month}-#{t.day}"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.r
|
157
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.b
|
161
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.w
|
165
|
+
"\e[1;37m"
|
166
|
+
end
|
167
|
+
|
168
|
+
def self.reset
|
169
|
+
@@b = @@r = @@day = nil
|
170
|
+
"\e[0m"
|
171
|
+
end
|
172
|
+
|
121
173
|
def self.banner
|
122
|
-
%
|
123
|
-
m,
|
124
|
-
`$b
|
125
|
-
.ss, $$: .,d$
|
126
|
-
`$$P,d$P' .,md$P"'
|
127
|
-
,$$$$$
|
128
|
-
.d
|
129
|
-
$$^' `"
|
130
|
-
$:
|
131
|
-
`b :$$
|
132
|
-
$$:
|
133
|
-
$$ |____/|_
|
134
|
-
.d$$
|
135
|
-
}
|
174
|
+
%{
|
175
|
+
#{w} m,
|
176
|
+
#{w} `$b
|
177
|
+
#{w} .ss, $$: .,d$
|
178
|
+
#{w} `$$P,d$P' .,md$P"'
|
179
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
180
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
181
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
182
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
183
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
184
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
185
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
186
|
+
#{w} .d$$ #{r} |_|
|
187
|
+
#{reset}}
|
136
188
|
end
|
137
189
|
|
138
190
|
SIGNAL_HANDLERS = {
|
139
191
|
# Ctrl-C in terminal
|
140
|
-
|
192
|
+
"INT" => ->(cli) { raise Interrupt },
|
141
193
|
# TERM is the signal that Sidekiq must exit.
|
142
194
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
cli.launcher.quiet
|
147
|
-
},
|
148
|
-
'TSTP' => ->(cli) {
|
149
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
195
|
+
"TERM" => ->(cli) { raise Interrupt },
|
196
|
+
"TSTP" => ->(cli) {
|
197
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
150
198
|
cli.launcher.quiet
|
151
199
|
},
|
152
|
-
|
153
|
-
if Sidekiq.options[:logfile]
|
154
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
155
|
-
Sidekiq::Logging.reopen_logs
|
156
|
-
end
|
157
|
-
},
|
158
|
-
'TTIN' => ->(cli) {
|
200
|
+
"TTIN" => ->(cli) {
|
159
201
|
Thread.list.each do |thread|
|
160
|
-
|
202
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
161
203
|
if thread.backtrace
|
162
|
-
|
204
|
+
cli.logger.warn thread.backtrace.join("\n")
|
163
205
|
else
|
164
|
-
|
206
|
+
cli.logger.warn "<no backtrace available>"
|
165
207
|
end
|
166
208
|
end
|
167
|
-
}
|
209
|
+
}
|
168
210
|
}
|
211
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
212
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
169
213
|
|
170
214
|
def handle_signal(sig)
|
171
|
-
|
172
|
-
|
173
|
-
if handy
|
174
|
-
handy.call(self)
|
175
|
-
else
|
176
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
177
|
-
end
|
215
|
+
logger.debug "Got #{sig} signal"
|
216
|
+
SIGNAL_HANDLERS[sig].call(self)
|
178
217
|
end
|
179
218
|
|
180
219
|
private
|
181
220
|
|
182
221
|
def print_banner
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
puts Sidekiq::CLI.banner
|
187
|
-
puts "\e[0m"
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
def daemonize
|
192
|
-
return unless options[:daemon]
|
193
|
-
|
194
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
195
|
-
files_to_reopen = []
|
196
|
-
ObjectSpace.each_object(File) do |file|
|
197
|
-
files_to_reopen << file unless file.closed?
|
198
|
-
end
|
199
|
-
|
200
|
-
::Process.daemon(true, true)
|
201
|
-
|
202
|
-
files_to_reopen.each do |file|
|
203
|
-
begin
|
204
|
-
file.reopen file.path, "a+"
|
205
|
-
file.sync = true
|
206
|
-
rescue ::Exception
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
|
-
[$stdout, $stderr].each do |io|
|
211
|
-
File.open(options[:logfile], 'ab') do |f|
|
212
|
-
io.reopen(f)
|
213
|
-
end
|
214
|
-
io.sync = true
|
215
|
-
end
|
216
|
-
$stdin.reopen('/dev/null')
|
217
|
-
|
218
|
-
initialize_logger
|
222
|
+
puts "\e[31m"
|
223
|
+
puts Sidekiq::CLI.banner
|
224
|
+
puts "\e[0m"
|
219
225
|
end
|
220
226
|
|
221
227
|
def set_environment(cli_env)
|
222
|
-
|
228
|
+
# See #984 for discussion.
|
229
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
230
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
231
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
232
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
233
|
+
config[:environment] = @environment
|
223
234
|
end
|
224
235
|
|
225
236
|
def symbolize_keys_deep!(hash)
|
226
237
|
hash.keys.each do |k|
|
227
238
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
228
239
|
hash[symkey] = hash.delete k
|
229
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
240
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
230
241
|
end
|
231
242
|
end
|
232
243
|
|
@@ -234,58 +245,74 @@ module Sidekiq
|
|
234
245
|
alias_method :☠, :exit
|
235
246
|
|
236
247
|
def setup_options(args)
|
248
|
+
# parse CLI options
|
237
249
|
opts = parse_options(args)
|
250
|
+
|
238
251
|
set_environment opts[:environment]
|
239
252
|
|
240
|
-
|
241
|
-
|
253
|
+
# check config file presence
|
254
|
+
if opts[:config_file]
|
255
|
+
unless File.exist?(opts[:config_file])
|
256
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
257
|
+
end
|
258
|
+
else
|
259
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
260
|
+
File.join(opts[:require], "config")
|
261
|
+
else
|
262
|
+
File.join(@config[:require], "config")
|
263
|
+
end
|
242
264
|
|
243
|
-
|
244
|
-
|
265
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
266
|
+
path = File.join(config_dir, config_file)
|
267
|
+
opts[:config_file] ||= path if File.exist?(path)
|
268
|
+
end
|
269
|
+
end
|
245
270
|
|
246
|
-
options
|
247
|
-
|
271
|
+
# parse config file options
|
272
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
273
|
+
|
274
|
+
# set defaults
|
275
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
276
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
277
|
+
|
278
|
+
# merge with defaults
|
279
|
+
@config.merge!(opts)
|
248
280
|
|
249
|
-
|
250
|
-
|
281
|
+
@config.default_capsule.tap do |cap|
|
282
|
+
cap.queues = opts[:queues]
|
283
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
284
|
+
end
|
285
|
+
|
286
|
+
opts[:capsules]&.each do |name, cap_config|
|
287
|
+
@config.capsule(name.to_s) do |cap|
|
288
|
+
cap.queues = cap_config[:queues]
|
289
|
+
cap.concurrency = cap_config[:concurrency]
|
290
|
+
end
|
291
|
+
end
|
251
292
|
end
|
252
293
|
|
253
|
-
def
|
254
|
-
ENV[
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
if ::Rails::VERSION::MAJOR < 4
|
261
|
-
raise "Sidekiq no longer supports this version of Rails"
|
262
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
263
|
-
# Painful contortions, see 1791 for discussion
|
264
|
-
# No autoloading, we want to force eager load for everything.
|
265
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
266
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
267
|
-
::Rails.application.config.eager_load = true
|
268
|
-
end
|
269
|
-
require 'sidekiq/rails'
|
270
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
271
|
-
else
|
272
|
-
require 'sidekiq/rails'
|
273
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
294
|
+
def boot_application
|
295
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
296
|
+
|
297
|
+
if File.directory?(@config[:require])
|
298
|
+
require "rails"
|
299
|
+
if ::Rails::VERSION::MAJOR < 6
|
300
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
274
301
|
end
|
275
|
-
|
302
|
+
require "sidekiq/rails"
|
303
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
304
|
+
@config[:tag] ||= default_tag
|
276
305
|
else
|
277
|
-
|
278
|
-
"./#{options[:require]} or /path/to/#{options[:require]}"
|
279
|
-
|
280
|
-
require(options[:require]) || raise(ArgumentError, not_required_message)
|
306
|
+
require @config[:require]
|
281
307
|
end
|
282
308
|
end
|
283
309
|
|
284
310
|
def default_tag
|
285
311
|
dir = ::Rails.root
|
286
312
|
name = File.basename(dir)
|
287
|
-
|
288
|
-
|
313
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
314
|
+
if name.to_i != 0 && prevdir
|
315
|
+
if File.basename(prevdir) == "releases"
|
289
316
|
return File.basename(File.dirname(prevdir))
|
290
317
|
end
|
291
318
|
end
|
@@ -293,57 +320,52 @@ module Sidekiq
|
|
293
320
|
end
|
294
321
|
|
295
322
|
def validate!
|
296
|
-
|
297
|
-
|
298
|
-
if !File.exist?(options[:require]) ||
|
299
|
-
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
323
|
+
if !File.exist?(@config[:require]) ||
|
324
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
300
325
|
logger.info "=================================================================="
|
301
|
-
logger.info " Please point
|
302
|
-
logger.info " to load your
|
326
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
327
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
303
328
|
logger.info "=================================================================="
|
304
329
|
logger.info @parser
|
305
330
|
die(1)
|
306
331
|
end
|
307
332
|
|
308
333
|
[:concurrency, :timeout].each do |opt|
|
309
|
-
raise ArgumentError, "#{opt}: #{
|
334
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
310
335
|
end
|
311
336
|
end
|
312
337
|
|
313
338
|
def parse_options(argv)
|
314
339
|
opts = {}
|
340
|
+
@parser = option_parser(opts)
|
341
|
+
@parser.parse!(argv)
|
342
|
+
opts
|
343
|
+
end
|
315
344
|
|
316
|
-
|
317
|
-
|
345
|
+
def option_parser(opts)
|
346
|
+
parser = OptionParser.new { |o|
|
347
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
318
348
|
opts[:concurrency] = Integer(arg)
|
319
349
|
end
|
320
350
|
|
321
|
-
o.on
|
322
|
-
opts[:daemon] = arg
|
323
|
-
end
|
324
|
-
|
325
|
-
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
351
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
326
352
|
opts[:environment] = arg
|
327
353
|
end
|
328
354
|
|
329
|
-
o.on
|
355
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
330
356
|
opts[:tag] = arg
|
331
357
|
end
|
332
358
|
|
333
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
334
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
335
|
-
end
|
336
|
-
|
337
359
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
338
|
-
|
339
|
-
|
360
|
+
opts[:queues] ||= []
|
361
|
+
opts[:queues] << arg
|
340
362
|
end
|
341
363
|
|
342
|
-
o.on
|
364
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
343
365
|
opts[:require] = arg
|
344
366
|
end
|
345
367
|
|
346
|
-
o.on
|
368
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
347
369
|
opts[:timeout] = Integer(arg)
|
348
370
|
end
|
349
371
|
|
@@ -351,88 +373,51 @@ module Sidekiq
|
|
351
373
|
opts[:verbose] = arg
|
352
374
|
end
|
353
375
|
|
354
|
-
o.on
|
376
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
355
377
|
opts[:config_file] = arg
|
356
378
|
end
|
357
379
|
|
358
|
-
o.on
|
359
|
-
opts[:logfile] = arg
|
360
|
-
end
|
361
|
-
|
362
|
-
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
363
|
-
opts[:pidfile] = arg
|
364
|
-
end
|
365
|
-
|
366
|
-
o.on '-V', '--version', "Print version and exit" do |arg|
|
380
|
+
o.on "-V", "--version", "Print version and exit" do
|
367
381
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
368
382
|
die(0)
|
369
383
|
end
|
370
|
-
|
384
|
+
}
|
371
385
|
|
372
|
-
|
373
|
-
|
374
|
-
logger.info
|
386
|
+
parser.banner = "sidekiq [options]"
|
387
|
+
parser.on_tail "-h", "--help", "Show help" do
|
388
|
+
logger.info parser
|
375
389
|
die 1
|
376
390
|
end
|
377
|
-
@parser.parse!(argv)
|
378
391
|
|
379
|
-
|
380
|
-
opts[:config_file] ||= filename if File.exist?(filename)
|
381
|
-
end
|
382
|
-
|
383
|
-
opts
|
392
|
+
parser
|
384
393
|
end
|
385
394
|
|
386
395
|
def initialize_logger
|
387
|
-
|
388
|
-
|
389
|
-
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
390
|
-
end
|
391
|
-
|
392
|
-
def write_pid
|
393
|
-
if path = options[:pidfile]
|
394
|
-
pidfile = File.expand_path(path)
|
395
|
-
File.open(pidfile, 'w') do |f|
|
396
|
-
f.puts ::Process.pid
|
397
|
-
end
|
398
|
-
end
|
396
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
399
397
|
end
|
400
398
|
|
401
|
-
def parse_config(
|
402
|
-
|
403
|
-
|
404
|
-
|
399
|
+
def parse_config(path)
|
400
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
401
|
+
erb.filename = File.expand_path(path)
|
402
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
405
403
|
|
406
|
-
|
407
|
-
|
408
|
-
else
|
409
|
-
symbolize_keys_deep!(opts)
|
410
|
-
end
|
411
|
-
|
412
|
-
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
413
|
-
parse_queues(opts, opts.delete(:queues) || [])
|
404
|
+
if opts.respond_to? :deep_symbolize_keys!
|
405
|
+
opts.deep_symbolize_keys!
|
414
406
|
else
|
415
|
-
|
416
|
-
# can be deployed by cap with just the defaults.
|
417
|
-
end
|
418
|
-
ns = opts.delete(:namespace)
|
419
|
-
if ns
|
420
|
-
# logger hasn't been initialized yet, puts is all we have.
|
421
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
422
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
407
|
+
symbolize_keys_deep!(opts)
|
423
408
|
end
|
424
|
-
opts
|
425
|
-
end
|
426
409
|
|
427
|
-
|
428
|
-
|
410
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
411
|
+
opts.delete(:strict)
|
412
|
+
|
413
|
+
opts
|
429
414
|
end
|
430
415
|
|
431
|
-
def
|
432
|
-
|
433
|
-
(opts[:queues] ||= []) << q
|
434
|
-
end
|
435
|
-
opts[:strict] = false if weight.to_i > 0
|
416
|
+
def rails_app?
|
417
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
436
418
|
end
|
437
419
|
end
|
438
420
|
end
|
421
|
+
|
422
|
+
require "sidekiq/systemd"
|
423
|
+
require "sidekiq/metrics/tracking"
|