sidekiq 5.2.2 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +657 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +558 -353
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +238 -260
- data/lib/sidekiq/client.rb +127 -102
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +35 -9
- data/lib/sidekiq/job_retry.rb +162 -102
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +203 -105
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -46
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +127 -118
- data/lib/sidekiq/rails.rb +50 -39
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +40 -89
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +111 -49
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +90 -89
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +189 -79
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +160 -114
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +68 -107
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +94 -182
- data/sidekiq.gemspec +25 -18
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +146 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +35 -283
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +143 -521
- data/web/assets/stylesheets/bootstrap.css +1 -1
- data/web/locales/ar.yml +71 -65
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +86 -66
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +6 -3
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +77 -27
- data/web/views/dashboard.erb +48 -18
- data/web/views/dead.erb +3 -3
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -24
- data/web/views/queues.erb +19 -5
- data/web/views/retries.erb +16 -17
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +80 -65
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -221
- data/Gemfile +0 -14
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -746
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/delay.rb +0 -42
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -204
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,45 +1,34 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
$stdout.sync = true
|
3
4
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
9
10
|
|
10
|
-
require
|
11
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
13
|
+
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
15
|
+
require "sidekiq/launcher"
|
12
16
|
|
13
|
-
module Sidekiq
|
17
|
+
module Sidekiq # :nodoc:
|
14
18
|
class CLI
|
15
|
-
include
|
19
|
+
include Sidekiq::Component
|
16
20
|
include Singleton unless $TESTING
|
17
21
|
|
18
|
-
PROCTITLES = [
|
19
|
-
proc { 'sidekiq' },
|
20
|
-
proc { Sidekiq::VERSION },
|
21
|
-
proc { |me, data| data['tag'] },
|
22
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
23
|
-
proc { |me, data| "stopping" if me.stopping? },
|
24
|
-
]
|
25
|
-
|
26
|
-
# Used for CLI testing
|
27
|
-
attr_accessor :code
|
28
22
|
attr_accessor :launcher
|
29
23
|
attr_accessor :environment
|
24
|
+
attr_accessor :config
|
30
25
|
|
31
|
-
def
|
32
|
-
@
|
33
|
-
end
|
34
|
-
|
35
|
-
def parse(args=ARGV)
|
36
|
-
@code = nil
|
26
|
+
def parse(args = ARGV.dup)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
37
28
|
|
38
29
|
setup_options(args)
|
39
30
|
initialize_logger
|
40
31
|
validate!
|
41
|
-
daemonize
|
42
|
-
write_pid
|
43
32
|
end
|
44
33
|
|
45
34
|
def jruby?
|
@@ -49,189 +38,206 @@ module Sidekiq
|
|
49
38
|
# Code within this method is not tested because it alters
|
50
39
|
# global process state irreversibly. PRs which improve the
|
51
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
52
|
-
def run
|
53
|
-
|
54
|
-
print_banner
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
55
43
|
|
56
|
-
|
57
|
-
|
58
|
-
# USR1 and USR2 don't work on the JVM
|
59
|
-
if !jruby?
|
60
|
-
sigs << 'USR1'
|
61
|
-
sigs << 'USR2'
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
|
+
print_banner
|
62
46
|
end
|
47
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
63
48
|
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
64
53
|
sigs.each do |sig|
|
65
|
-
|
66
|
-
|
67
|
-
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
68
62
|
end
|
69
|
-
|
70
|
-
puts "Signal #{sig} not supported"
|
63
|
+
self_write.puts(sig)
|
71
64
|
end
|
65
|
+
rescue ArgumentError
|
66
|
+
puts "Signal #{sig} not supported"
|
72
67
|
end
|
73
68
|
|
74
69
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
75
70
|
logger.info Sidekiq::LICENSE
|
76
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
71
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
77
72
|
|
78
73
|
# touch the connection pool so it is created before we
|
79
74
|
# fire startup and start multithreading.
|
80
|
-
|
81
|
-
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
78
|
+
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
81
|
+
# Redis Enterprise Cloud returns "" for their policy 😳
|
82
|
+
logger.warn <<~EOM
|
83
|
+
|
84
|
+
|
85
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
86
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
87
|
+
See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
|
88
|
+
|
89
|
+
EOM
|
90
|
+
end
|
82
91
|
|
83
92
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
84
93
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
85
|
-
|
86
|
-
|
87
|
-
|
94
|
+
@config.capsules.each_pair do |name, cap|
|
95
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
96
|
+
end
|
88
97
|
|
89
98
|
# cache process identity
|
90
|
-
|
99
|
+
@config[:identity] = identity
|
91
100
|
|
92
101
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
93
|
-
|
102
|
+
@config.server_middleware
|
94
103
|
|
95
104
|
# Before this point, the process is initializing with just the main thread.
|
96
105
|
# Starting here the process will now have multiple threads running.
|
97
106
|
fire_event(:startup, reverse: false, reraise: true)
|
98
107
|
|
99
|
-
logger.debug { "Client Middleware: #{
|
100
|
-
logger.debug { "Server Middleware: #{
|
108
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
109
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
110
|
+
|
111
|
+
launch(self_read)
|
112
|
+
end
|
101
113
|
|
102
|
-
|
103
|
-
|
114
|
+
def launch(self_read)
|
115
|
+
if environment == "development" && $stdout.tty?
|
116
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
104
117
|
end
|
105
118
|
|
106
|
-
|
107
|
-
@launcher = Sidekiq::Launcher.new(options)
|
119
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
108
120
|
|
109
121
|
begin
|
110
122
|
launcher.run
|
111
123
|
|
112
|
-
while
|
113
|
-
signal =
|
124
|
+
while self_read.wait_readable
|
125
|
+
signal = self_read.gets.strip
|
114
126
|
handle_signal(signal)
|
115
127
|
end
|
116
128
|
rescue Interrupt
|
117
|
-
logger.info
|
129
|
+
logger.info "Shutting down"
|
118
130
|
launcher.stop
|
119
|
-
# Explicitly exit so busy Processor threads can't block
|
120
|
-
# process shutdown.
|
121
131
|
logger.info "Bye!"
|
132
|
+
|
133
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
134
|
+
#
|
135
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
136
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
137
|
+
# use the TTIN signal to determine where things are stuck.
|
122
138
|
exit(0)
|
123
139
|
end
|
124
140
|
end
|
125
141
|
|
142
|
+
HOLIDAY_COLORS = {
|
143
|
+
# got other color-specific holidays from around the world?
|
144
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
145
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
146
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
147
|
+
}
|
148
|
+
|
149
|
+
def self.day
|
150
|
+
@@day ||= begin
|
151
|
+
t = Date.today
|
152
|
+
"#{t.month}-#{t.day}"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.r
|
157
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.b
|
161
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.w
|
165
|
+
"\e[1;37m"
|
166
|
+
end
|
167
|
+
|
168
|
+
def self.reset
|
169
|
+
@@b = @@r = @@day = nil
|
170
|
+
"\e[0m"
|
171
|
+
end
|
172
|
+
|
126
173
|
def self.banner
|
127
|
-
%
|
128
|
-
m,
|
129
|
-
`$b
|
130
|
-
.ss, $$: .,d$
|
131
|
-
`$$P,d$P' .,md$P"'
|
132
|
-
,$$$$$
|
133
|
-
.d
|
134
|
-
$$^' `"
|
135
|
-
$:
|
136
|
-
`b :$$
|
137
|
-
$$:
|
138
|
-
$$ |____/|_
|
139
|
-
.d$$
|
140
|
-
}
|
174
|
+
%{
|
175
|
+
#{w} m,
|
176
|
+
#{w} `$b
|
177
|
+
#{w} .ss, $$: .,d$
|
178
|
+
#{w} `$$P,d$P' .,md$P"'
|
179
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
180
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
181
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
182
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
183
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
184
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
185
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
186
|
+
#{w} .d$$ #{r} |_|
|
187
|
+
#{reset}}
|
141
188
|
end
|
142
189
|
|
143
190
|
SIGNAL_HANDLERS = {
|
144
191
|
# Ctrl-C in terminal
|
145
|
-
|
192
|
+
"INT" => ->(cli) { raise Interrupt },
|
146
193
|
# TERM is the signal that Sidekiq must exit.
|
147
194
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
148
|
-
|
149
|
-
|
150
|
-
|
195
|
+
"TERM" => ->(cli) { raise Interrupt },
|
196
|
+
"TSTP" => ->(cli) {
|
197
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
151
198
|
cli.launcher.quiet
|
152
199
|
},
|
153
|
-
|
154
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
155
|
-
cli.launcher.quiet
|
156
|
-
},
|
157
|
-
'USR2' => ->(cli) {
|
158
|
-
if Sidekiq.options[:logfile]
|
159
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
160
|
-
Sidekiq::Logging.reopen_logs
|
161
|
-
end
|
162
|
-
},
|
163
|
-
'TTIN' => ->(cli) {
|
200
|
+
"TTIN" => ->(cli) {
|
164
201
|
Thread.list.each do |thread|
|
165
|
-
|
202
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
166
203
|
if thread.backtrace
|
167
|
-
|
204
|
+
cli.logger.warn thread.backtrace.join("\n")
|
168
205
|
else
|
169
|
-
|
206
|
+
cli.logger.warn "<no backtrace available>"
|
170
207
|
end
|
171
208
|
end
|
172
|
-
}
|
209
|
+
}
|
173
210
|
}
|
211
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
212
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
174
213
|
|
175
214
|
def handle_signal(sig)
|
176
|
-
|
177
|
-
|
178
|
-
if handy
|
179
|
-
handy.call(self)
|
180
|
-
else
|
181
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
182
|
-
end
|
215
|
+
logger.debug "Got #{sig} signal"
|
216
|
+
SIGNAL_HANDLERS[sig].call(self)
|
183
217
|
end
|
184
218
|
|
185
219
|
private
|
186
220
|
|
187
221
|
def print_banner
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
puts Sidekiq::CLI.banner
|
192
|
-
puts "\e[0m"
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
def daemonize
|
197
|
-
return unless options[:daemon]
|
198
|
-
|
199
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
200
|
-
files_to_reopen = []
|
201
|
-
ObjectSpace.each_object(File) do |file|
|
202
|
-
files_to_reopen << file unless file.closed?
|
203
|
-
end
|
204
|
-
|
205
|
-
::Process.daemon(true, true)
|
206
|
-
|
207
|
-
files_to_reopen.each do |file|
|
208
|
-
begin
|
209
|
-
file.reopen file.path, "a+"
|
210
|
-
file.sync = true
|
211
|
-
rescue ::Exception
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
[$stdout, $stderr].each do |io|
|
216
|
-
File.open(options[:logfile], 'ab') do |f|
|
217
|
-
io.reopen(f)
|
218
|
-
end
|
219
|
-
io.sync = true
|
220
|
-
end
|
221
|
-
$stdin.reopen('/dev/null')
|
222
|
-
|
223
|
-
initialize_logger
|
222
|
+
puts "\e[31m"
|
223
|
+
puts Sidekiq::CLI.banner
|
224
|
+
puts "\e[0m"
|
224
225
|
end
|
225
226
|
|
226
227
|
def set_environment(cli_env)
|
227
|
-
|
228
|
+
# See #984 for discussion.
|
229
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
230
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
231
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
232
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
233
|
+
config[:environment] = @environment
|
228
234
|
end
|
229
235
|
|
230
236
|
def symbolize_keys_deep!(hash)
|
231
237
|
hash.keys.each do |k|
|
232
238
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
233
239
|
hash[symkey] = hash.delete k
|
234
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
240
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
235
241
|
end
|
236
242
|
end
|
237
243
|
|
@@ -239,58 +245,74 @@ module Sidekiq
|
|
239
245
|
alias_method :☠, :exit
|
240
246
|
|
241
247
|
def setup_options(args)
|
248
|
+
# parse CLI options
|
242
249
|
opts = parse_options(args)
|
250
|
+
|
243
251
|
set_environment opts[:environment]
|
244
252
|
|
245
|
-
|
246
|
-
|
253
|
+
# check config file presence
|
254
|
+
if opts[:config_file]
|
255
|
+
unless File.exist?(opts[:config_file])
|
256
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
257
|
+
end
|
258
|
+
else
|
259
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
260
|
+
File.join(opts[:require], "config")
|
261
|
+
else
|
262
|
+
File.join(@config[:require], "config")
|
263
|
+
end
|
247
264
|
|
248
|
-
|
249
|
-
|
265
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
266
|
+
path = File.join(config_dir, config_file)
|
267
|
+
opts[:config_file] ||= path if File.exist?(path)
|
268
|
+
end
|
269
|
+
end
|
250
270
|
|
251
|
-
options
|
252
|
-
|
271
|
+
# parse config file options
|
272
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
273
|
+
|
274
|
+
# set defaults
|
275
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
276
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
277
|
+
|
278
|
+
# merge with defaults
|
279
|
+
@config.merge!(opts)
|
253
280
|
|
254
|
-
|
255
|
-
|
281
|
+
@config.default_capsule.tap do |cap|
|
282
|
+
cap.queues = opts[:queues]
|
283
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
284
|
+
end
|
285
|
+
|
286
|
+
opts[:capsules]&.each do |name, cap_config|
|
287
|
+
@config.capsule(name.to_s) do |cap|
|
288
|
+
cap.queues = cap_config[:queues]
|
289
|
+
cap.concurrency = cap_config[:concurrency]
|
290
|
+
end
|
291
|
+
end
|
256
292
|
end
|
257
293
|
|
258
|
-
def
|
259
|
-
ENV[
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
if ::Rails::VERSION::MAJOR < 4
|
266
|
-
raise "Sidekiq no longer supports this version of Rails"
|
267
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
268
|
-
# Painful contortions, see 1791 for discussion
|
269
|
-
# No autoloading, we want to force eager load for everything.
|
270
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
271
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
272
|
-
::Rails.application.config.eager_load = true
|
273
|
-
end
|
274
|
-
require 'sidekiq/rails'
|
275
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
276
|
-
else
|
277
|
-
require 'sidekiq/rails'
|
278
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
294
|
+
def boot_application
|
295
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
296
|
+
|
297
|
+
if File.directory?(@config[:require])
|
298
|
+
require "rails"
|
299
|
+
if ::Rails::VERSION::MAJOR < 6
|
300
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
279
301
|
end
|
280
|
-
|
302
|
+
require "sidekiq/rails"
|
303
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
304
|
+
@config[:tag] ||= default_tag
|
281
305
|
else
|
282
|
-
|
283
|
-
"./#{options[:require]} or /path/to/#{options[:require]}"
|
284
|
-
|
285
|
-
require(options[:require]) || raise(ArgumentError, not_required_message)
|
306
|
+
require @config[:require]
|
286
307
|
end
|
287
308
|
end
|
288
309
|
|
289
310
|
def default_tag
|
290
311
|
dir = ::Rails.root
|
291
312
|
name = File.basename(dir)
|
292
|
-
|
293
|
-
|
313
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
314
|
+
if name.to_i != 0 && prevdir
|
315
|
+
if File.basename(prevdir) == "releases"
|
294
316
|
return File.basename(File.dirname(prevdir))
|
295
317
|
end
|
296
318
|
end
|
@@ -298,59 +320,52 @@ module Sidekiq
|
|
298
320
|
end
|
299
321
|
|
300
322
|
def validate!
|
301
|
-
|
302
|
-
|
303
|
-
if !File.exist?(options[:require]) ||
|
304
|
-
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
323
|
+
if !File.exist?(@config[:require]) ||
|
324
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
305
325
|
logger.info "=================================================================="
|
306
|
-
logger.info " Please point
|
307
|
-
logger.info " to load your
|
326
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
327
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
308
328
|
logger.info "=================================================================="
|
309
329
|
logger.info @parser
|
310
330
|
die(1)
|
311
331
|
end
|
312
332
|
|
313
333
|
[:concurrency, :timeout].each do |opt|
|
314
|
-
raise ArgumentError, "#{opt}: #{
|
334
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
315
335
|
end
|
316
336
|
end
|
317
337
|
|
318
338
|
def parse_options(argv)
|
319
339
|
opts = {}
|
340
|
+
@parser = option_parser(opts)
|
341
|
+
@parser.parse!(argv)
|
342
|
+
opts
|
343
|
+
end
|
320
344
|
|
321
|
-
|
322
|
-
|
345
|
+
def option_parser(opts)
|
346
|
+
parser = OptionParser.new { |o|
|
347
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
323
348
|
opts[:concurrency] = Integer(arg)
|
324
349
|
end
|
325
350
|
|
326
|
-
o.on
|
327
|
-
opts[:daemon] = arg
|
328
|
-
end
|
329
|
-
|
330
|
-
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
351
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
331
352
|
opts[:environment] = arg
|
332
353
|
end
|
333
354
|
|
334
|
-
o.on
|
355
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
335
356
|
opts[:tag] = arg
|
336
357
|
end
|
337
358
|
|
338
|
-
# this index remains here for backwards compatibility but none of the Sidekiq
|
339
|
-
# family use this value anymore. it was used by Pro's original reliable_fetch.
|
340
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
341
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
342
|
-
end
|
343
|
-
|
344
359
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
345
|
-
|
346
|
-
|
360
|
+
opts[:queues] ||= []
|
361
|
+
opts[:queues] << arg
|
347
362
|
end
|
348
363
|
|
349
|
-
o.on
|
364
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
350
365
|
opts[:require] = arg
|
351
366
|
end
|
352
367
|
|
353
|
-
o.on
|
368
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
354
369
|
opts[:timeout] = Integer(arg)
|
355
370
|
end
|
356
371
|
|
@@ -358,88 +373,51 @@ module Sidekiq
|
|
358
373
|
opts[:verbose] = arg
|
359
374
|
end
|
360
375
|
|
361
|
-
o.on
|
376
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
362
377
|
opts[:config_file] = arg
|
363
378
|
end
|
364
379
|
|
365
|
-
o.on
|
366
|
-
opts[:logfile] = arg
|
367
|
-
end
|
368
|
-
|
369
|
-
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
370
|
-
opts[:pidfile] = arg
|
371
|
-
end
|
372
|
-
|
373
|
-
o.on '-V', '--version', "Print version and exit" do |arg|
|
380
|
+
o.on "-V", "--version", "Print version and exit" do
|
374
381
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
375
382
|
die(0)
|
376
383
|
end
|
377
|
-
|
384
|
+
}
|
378
385
|
|
379
|
-
|
380
|
-
|
381
|
-
logger.info
|
386
|
+
parser.banner = "sidekiq [options]"
|
387
|
+
parser.on_tail "-h", "--help", "Show help" do
|
388
|
+
logger.info parser
|
382
389
|
die 1
|
383
390
|
end
|
384
|
-
@parser.parse!(argv)
|
385
391
|
|
386
|
-
|
387
|
-
opts[:config_file] ||= filename if File.exist?(filename)
|
388
|
-
end
|
389
|
-
|
390
|
-
opts
|
392
|
+
parser
|
391
393
|
end
|
392
394
|
|
393
395
|
def initialize_logger
|
394
|
-
|
395
|
-
|
396
|
-
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
396
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
397
397
|
end
|
398
398
|
|
399
|
-
def
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
f.puts ::Process.pid
|
404
|
-
end
|
405
|
-
end
|
406
|
-
end
|
407
|
-
|
408
|
-
def parse_config(cfile)
|
409
|
-
opts = {}
|
410
|
-
if File.exist?(cfile)
|
411
|
-
opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
|
412
|
-
|
413
|
-
if opts.respond_to? :deep_symbolize_keys!
|
414
|
-
opts.deep_symbolize_keys!
|
415
|
-
else
|
416
|
-
symbolize_keys_deep!(opts)
|
417
|
-
end
|
399
|
+
def parse_config(path)
|
400
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
401
|
+
erb.filename = File.expand_path(path)
|
402
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
418
403
|
|
419
|
-
|
420
|
-
|
404
|
+
if opts.respond_to? :deep_symbolize_keys!
|
405
|
+
opts.deep_symbolize_keys!
|
421
406
|
else
|
422
|
-
|
423
|
-
# can be deployed by cap with just the defaults.
|
407
|
+
symbolize_keys_deep!(opts)
|
424
408
|
end
|
425
|
-
ns = opts.delete(:namespace)
|
426
|
-
if ns
|
427
|
-
# logger hasn't been initialized yet, puts is all we have.
|
428
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
429
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
430
|
-
end
|
431
|
-
opts
|
432
|
-
end
|
433
409
|
|
434
|
-
|
435
|
-
|
410
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
411
|
+
opts.delete(:strict)
|
412
|
+
|
413
|
+
opts
|
436
414
|
end
|
437
415
|
|
438
|
-
def
|
439
|
-
|
440
|
-
raise ArgumentError, "queues: #{q} cannot be defined twice" if opts[:queues].include?(q)
|
441
|
-
[weight.to_i, 1].max.times { opts[:queues] << q }
|
442
|
-
opts[:strict] = false if weight.to_i > 0
|
416
|
+
def rails_app?
|
417
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
443
418
|
end
|
444
419
|
end
|
445
420
|
end
|
421
|
+
|
422
|
+
require "sidekiq/systemd"
|
423
|
+
require "sidekiq/metrics/tracking"
|