sidekiq 5.2.7 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +619 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +558 -353
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +204 -226
- data/lib/sidekiq/client.rb +127 -102
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +147 -108
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +203 -105
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -46
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +122 -120
- data/lib/sidekiq/rails.rb +48 -38
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +39 -107
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +111 -49
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +90 -89
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +190 -80
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +154 -115
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +68 -107
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +92 -182
- data/sidekiq.gemspec +25 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +146 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +35 -293
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +111 -522
- data/web/locales/ar.yml +71 -65
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +86 -66
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +6 -3
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +77 -27
- data/web/views/dashboard.erb +48 -18
- data/web/views/dead.erb +3 -3
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -24
- data/web/views/queues.erb +19 -5
- data/web/views/retries.erb +16 -17
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +70 -70
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -23
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/delay.rb +0 -42
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -220
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,33 +1,31 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
$stdout.sync = true
|
3
4
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
9
10
|
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
13
|
+
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
15
|
+
require "sidekiq/launcher"
|
13
16
|
|
14
|
-
module Sidekiq
|
17
|
+
module Sidekiq # :nodoc:
|
15
18
|
class CLI
|
16
|
-
include
|
19
|
+
include Sidekiq::Component
|
17
20
|
include Singleton unless $TESTING
|
18
21
|
|
19
|
-
PROCTITLES = [
|
20
|
-
proc { 'sidekiq' },
|
21
|
-
proc { Sidekiq::VERSION },
|
22
|
-
proc { |me, data| data['tag'] },
|
23
|
-
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
-
proc { |me, data| "stopping" if me.stopping? },
|
25
|
-
]
|
26
|
-
|
27
22
|
attr_accessor :launcher
|
28
23
|
attr_accessor :environment
|
24
|
+
attr_accessor :config
|
25
|
+
|
26
|
+
def parse(args = ARGV.dup)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
29
28
|
|
30
|
-
def parse(args = ARGV)
|
31
29
|
setup_options(args)
|
32
30
|
initialize_logger
|
33
31
|
validate!
|
@@ -40,187 +38,206 @@ module Sidekiq
|
|
40
38
|
# Code within this method is not tested because it alters
|
41
39
|
# global process state irreversibly. PRs which improve the
|
42
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
43
|
-
def run
|
44
|
-
|
45
|
-
write_pid
|
46
|
-
boot_system
|
47
|
-
print_banner if environment == 'development' && $stdout.tty?
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
48
43
|
|
49
|
-
|
50
|
-
|
51
|
-
# USR1 and USR2 don't work on the JVM
|
52
|
-
if !jruby?
|
53
|
-
sigs << 'USR1'
|
54
|
-
sigs << 'USR2'
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
|
+
print_banner
|
55
46
|
end
|
47
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
56
48
|
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
57
53
|
sigs.each do |sig|
|
58
|
-
|
59
|
-
|
60
|
-
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
61
62
|
end
|
62
|
-
|
63
|
-
puts "Signal #{sig} not supported"
|
63
|
+
self_write.puts(sig)
|
64
64
|
end
|
65
|
+
rescue ArgumentError
|
66
|
+
puts "Signal #{sig} not supported"
|
65
67
|
end
|
66
68
|
|
67
69
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
68
70
|
logger.info Sidekiq::LICENSE
|
69
|
-
logger.info "Upgrade to Sidekiq Pro for more features and support:
|
71
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
70
72
|
|
71
73
|
# touch the connection pool so it is created before we
|
72
74
|
# fire startup and start multithreading.
|
73
|
-
|
74
|
-
|
75
|
-
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
78
|
+
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
81
|
+
# Redis Enterprise Cloud returns "" for their policy 😳
|
82
|
+
logger.warn <<~EOM
|
83
|
+
|
84
|
+
|
85
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
86
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
87
|
+
See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
|
88
|
+
|
89
|
+
EOM
|
90
|
+
end
|
76
91
|
|
77
92
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
78
93
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
79
|
-
|
80
|
-
|
81
|
-
|
94
|
+
@config.capsules.each_pair do |name, cap|
|
95
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
96
|
+
end
|
82
97
|
|
83
98
|
# cache process identity
|
84
|
-
|
99
|
+
@config[:identity] = identity
|
85
100
|
|
86
101
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
87
|
-
|
102
|
+
@config.server_middleware
|
88
103
|
|
89
104
|
# Before this point, the process is initializing with just the main thread.
|
90
105
|
# Starting here the process will now have multiple threads running.
|
91
106
|
fire_event(:startup, reverse: false, reraise: true)
|
92
107
|
|
93
|
-
logger.debug { "Client Middleware: #{
|
94
|
-
logger.debug { "Server Middleware: #{
|
108
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
109
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
95
110
|
|
96
111
|
launch(self_read)
|
97
112
|
end
|
98
113
|
|
99
114
|
def launch(self_read)
|
100
|
-
if
|
101
|
-
logger.info
|
115
|
+
if environment == "development" && $stdout.tty?
|
116
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
102
117
|
end
|
103
118
|
|
104
|
-
@launcher = Sidekiq::Launcher.new(
|
119
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
105
120
|
|
106
121
|
begin
|
107
122
|
launcher.run
|
108
123
|
|
109
|
-
while
|
110
|
-
signal =
|
124
|
+
while self_read.wait_readable
|
125
|
+
signal = self_read.gets.strip
|
111
126
|
handle_signal(signal)
|
112
127
|
end
|
113
128
|
rescue Interrupt
|
114
|
-
logger.info
|
129
|
+
logger.info "Shutting down"
|
115
130
|
launcher.stop
|
116
|
-
# Explicitly exit so busy Processor threads can't block
|
117
|
-
# process shutdown.
|
118
131
|
logger.info "Bye!"
|
132
|
+
|
133
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
134
|
+
#
|
135
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
136
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
137
|
+
# use the TTIN signal to determine where things are stuck.
|
119
138
|
exit(0)
|
120
139
|
end
|
121
140
|
end
|
122
141
|
|
142
|
+
HOLIDAY_COLORS = {
|
143
|
+
# got other color-specific holidays from around the world?
|
144
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
145
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
146
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
147
|
+
}
|
148
|
+
|
149
|
+
def self.day
|
150
|
+
@@day ||= begin
|
151
|
+
t = Date.today
|
152
|
+
"#{t.month}-#{t.day}"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.r
|
157
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.b
|
161
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.w
|
165
|
+
"\e[1;37m"
|
166
|
+
end
|
167
|
+
|
168
|
+
def self.reset
|
169
|
+
@@b = @@r = @@day = nil
|
170
|
+
"\e[0m"
|
171
|
+
end
|
172
|
+
|
123
173
|
def self.banner
|
124
|
-
%
|
125
|
-
m,
|
126
|
-
`$b
|
127
|
-
.ss, $$: .,d$
|
128
|
-
`$$P,d$P' .,md$P"'
|
129
|
-
,$$$$$
|
130
|
-
.d
|
131
|
-
$$^' `"
|
132
|
-
$:
|
133
|
-
`b :$$
|
134
|
-
$$:
|
135
|
-
$$ |____/|_
|
136
|
-
.d$$
|
137
|
-
}
|
174
|
+
%{
|
175
|
+
#{w} m,
|
176
|
+
#{w} `$b
|
177
|
+
#{w} .ss, $$: .,d$
|
178
|
+
#{w} `$$P,d$P' .,md$P"'
|
179
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
180
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
181
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
182
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
183
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
184
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
185
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
186
|
+
#{w} .d$$ #{r} |_|
|
187
|
+
#{reset}}
|
138
188
|
end
|
139
189
|
|
140
190
|
SIGNAL_HANDLERS = {
|
141
191
|
# Ctrl-C in terminal
|
142
|
-
|
192
|
+
"INT" => ->(cli) { raise Interrupt },
|
143
193
|
# TERM is the signal that Sidekiq must exit.
|
144
194
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
145
|
-
|
146
|
-
|
147
|
-
|
195
|
+
"TERM" => ->(cli) { raise Interrupt },
|
196
|
+
"TSTP" => ->(cli) {
|
197
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
148
198
|
cli.launcher.quiet
|
149
199
|
},
|
150
|
-
|
151
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
152
|
-
cli.launcher.quiet
|
153
|
-
},
|
154
|
-
'USR2' => ->(cli) {
|
155
|
-
if Sidekiq.options[:logfile]
|
156
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
157
|
-
Sidekiq::Logging.reopen_logs
|
158
|
-
end
|
159
|
-
},
|
160
|
-
'TTIN' => ->(cli) {
|
200
|
+
"TTIN" => ->(cli) {
|
161
201
|
Thread.list.each do |thread|
|
162
|
-
|
202
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
163
203
|
if thread.backtrace
|
164
|
-
|
204
|
+
cli.logger.warn thread.backtrace.join("\n")
|
165
205
|
else
|
166
|
-
|
206
|
+
cli.logger.warn "<no backtrace available>"
|
167
207
|
end
|
168
208
|
end
|
169
|
-
}
|
209
|
+
}
|
170
210
|
}
|
211
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
212
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
171
213
|
|
172
214
|
def handle_signal(sig)
|
173
|
-
|
174
|
-
|
175
|
-
if handy
|
176
|
-
handy.call(self)
|
177
|
-
else
|
178
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
179
|
-
end
|
215
|
+
logger.debug "Got #{sig} signal"
|
216
|
+
SIGNAL_HANDLERS[sig].call(self)
|
180
217
|
end
|
181
218
|
|
182
219
|
private
|
183
220
|
|
184
221
|
def print_banner
|
185
|
-
puts "\e[
|
222
|
+
puts "\e[31m"
|
186
223
|
puts Sidekiq::CLI.banner
|
187
224
|
puts "\e[0m"
|
188
225
|
end
|
189
226
|
|
190
|
-
def daemonize
|
191
|
-
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
192
|
-
|
193
|
-
files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
|
194
|
-
::Process.daemon(true, true)
|
195
|
-
|
196
|
-
files_to_reopen.each do |file|
|
197
|
-
begin
|
198
|
-
file.reopen file.path, "a+"
|
199
|
-
file.sync = true
|
200
|
-
rescue ::Exception
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
[$stdout, $stderr].each do |io|
|
205
|
-
File.open(options[:logfile], 'ab') do |f|
|
206
|
-
io.reopen(f)
|
207
|
-
end
|
208
|
-
io.sync = true
|
209
|
-
end
|
210
|
-
$stdin.reopen('/dev/null')
|
211
|
-
|
212
|
-
initialize_logger
|
213
|
-
end
|
214
|
-
|
215
227
|
def set_environment(cli_env)
|
216
|
-
|
228
|
+
# See #984 for discussion.
|
229
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
230
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
231
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
232
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
233
|
+
config[:environment] = @environment
|
217
234
|
end
|
218
235
|
|
219
236
|
def symbolize_keys_deep!(hash)
|
220
237
|
hash.keys.each do |k|
|
221
238
|
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
222
239
|
hash[symkey] = hash.delete k
|
223
|
-
symbolize_keys_deep! hash[symkey] if hash[symkey].
|
240
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
224
241
|
end
|
225
242
|
end
|
226
243
|
|
@@ -235,14 +252,14 @@ module Sidekiq
|
|
235
252
|
|
236
253
|
# check config file presence
|
237
254
|
if opts[:config_file]
|
238
|
-
|
255
|
+
unless File.exist?(opts[:config_file])
|
239
256
|
raise ArgumentError, "No such file #{opts[:config_file]}"
|
240
257
|
end
|
241
258
|
else
|
242
259
|
config_dir = if File.directory?(opts[:require].to_s)
|
243
|
-
File.join(opts[:require],
|
260
|
+
File.join(opts[:require], "config")
|
244
261
|
else
|
245
|
-
File.join(
|
262
|
+
File.join(@config[:require], "config")
|
246
263
|
end
|
247
264
|
|
248
265
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -255,49 +272,47 @@ module Sidekiq
|
|
255
272
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
256
273
|
|
257
274
|
# set defaults
|
258
|
-
opts[:queues] =
|
259
|
-
opts[:strict] = true if opts[:strict].nil?
|
275
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
260
276
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
261
277
|
|
262
278
|
# merge with defaults
|
263
|
-
|
264
|
-
end
|
279
|
+
@config.merge!(opts)
|
265
280
|
|
266
|
-
|
267
|
-
|
281
|
+
@config.default_capsule.tap do |cap|
|
282
|
+
cap.queues = opts[:queues]
|
283
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
284
|
+
end
|
285
|
+
|
286
|
+
opts[:capsules]&.each do |name, cap_config|
|
287
|
+
@config.capsule(name.to_s) do |cap|
|
288
|
+
cap.queues = cap_config[:queues]
|
289
|
+
cap.concurrency = cap_config[:concurrency]
|
290
|
+
end
|
291
|
+
end
|
268
292
|
end
|
269
293
|
|
270
|
-
def
|
271
|
-
ENV[
|
272
|
-
|
273
|
-
if File.directory?(
|
274
|
-
require
|
275
|
-
if ::Rails::VERSION::MAJOR <
|
276
|
-
|
277
|
-
elsif ::Rails::VERSION::MAJOR == 4
|
278
|
-
# Painful contortions, see 1791 for discussion
|
279
|
-
# No autoloading, we want to force eager load for everything.
|
280
|
-
require File.expand_path("#{options[:require]}/config/application.rb")
|
281
|
-
::Rails::Application.initializer "sidekiq.eager_load" do
|
282
|
-
::Rails.application.config.eager_load = true
|
283
|
-
end
|
284
|
-
require 'sidekiq/rails'
|
285
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
286
|
-
else
|
287
|
-
require 'sidekiq/rails'
|
288
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
294
|
+
def boot_application
|
295
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
296
|
+
|
297
|
+
if File.directory?(@config[:require])
|
298
|
+
require "rails"
|
299
|
+
if ::Rails::VERSION::MAJOR < 6
|
300
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
289
301
|
end
|
290
|
-
|
302
|
+
require "sidekiq/rails"
|
303
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
304
|
+
@config[:tag] ||= default_tag
|
291
305
|
else
|
292
|
-
require
|
306
|
+
require @config[:require]
|
293
307
|
end
|
294
308
|
end
|
295
309
|
|
296
310
|
def default_tag
|
297
311
|
dir = ::Rails.root
|
298
312
|
name = File.basename(dir)
|
299
|
-
|
300
|
-
|
313
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
314
|
+
if name.to_i != 0 && prevdir
|
315
|
+
if File.basename(prevdir) == "releases"
|
301
316
|
return File.basename(File.dirname(prevdir))
|
302
317
|
end
|
303
318
|
end
|
@@ -305,58 +320,52 @@ module Sidekiq
|
|
305
320
|
end
|
306
321
|
|
307
322
|
def validate!
|
308
|
-
if !File.exist?(
|
309
|
-
|
323
|
+
if !File.exist?(@config[:require]) ||
|
324
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
310
325
|
logger.info "=================================================================="
|
311
|
-
logger.info " Please point
|
312
|
-
logger.info " to load your
|
326
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
327
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
313
328
|
logger.info "=================================================================="
|
314
329
|
logger.info @parser
|
315
330
|
die(1)
|
316
331
|
end
|
317
332
|
|
318
333
|
[:concurrency, :timeout].each do |opt|
|
319
|
-
raise ArgumentError, "#{opt}: #{
|
334
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
320
335
|
end
|
321
336
|
end
|
322
337
|
|
323
338
|
def parse_options(argv)
|
324
339
|
opts = {}
|
340
|
+
@parser = option_parser(opts)
|
341
|
+
@parser.parse!(argv)
|
342
|
+
opts
|
343
|
+
end
|
325
344
|
|
326
|
-
|
327
|
-
|
345
|
+
def option_parser(opts)
|
346
|
+
parser = OptionParser.new { |o|
|
347
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
328
348
|
opts[:concurrency] = Integer(arg)
|
329
349
|
end
|
330
350
|
|
331
|
-
o.on
|
332
|
-
opts[:daemon] = arg
|
333
|
-
puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
334
|
-
end
|
335
|
-
|
336
|
-
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
351
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
337
352
|
opts[:environment] = arg
|
338
353
|
end
|
339
354
|
|
340
|
-
o.on
|
355
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
341
356
|
opts[:tag] = arg
|
342
357
|
end
|
343
358
|
|
344
|
-
# this index remains here for backwards compatibility but none of the Sidekiq
|
345
|
-
# family use this value anymore. it was used by Pro's original reliable_fetch.
|
346
|
-
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
347
|
-
opts[:index] = Integer(arg.match(/\d+/)[0])
|
348
|
-
end
|
349
|
-
|
350
359
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
351
|
-
|
352
|
-
|
360
|
+
opts[:queues] ||= []
|
361
|
+
opts[:queues] << arg
|
353
362
|
end
|
354
363
|
|
355
|
-
o.on
|
364
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
356
365
|
opts[:require] = arg
|
357
366
|
end
|
358
367
|
|
359
|
-
o.on
|
368
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
360
369
|
opts[:timeout] = Integer(arg)
|
361
370
|
end
|
362
371
|
|
@@ -364,54 +373,33 @@ module Sidekiq
|
|
364
373
|
opts[:verbose] = arg
|
365
374
|
end
|
366
375
|
|
367
|
-
o.on
|
376
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
368
377
|
opts[:config_file] = arg
|
369
378
|
end
|
370
379
|
|
371
|
-
o.on
|
372
|
-
opts[:logfile] = arg
|
373
|
-
puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
|
374
|
-
end
|
375
|
-
|
376
|
-
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
377
|
-
opts[:pidfile] = arg
|
378
|
-
puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
|
379
|
-
end
|
380
|
-
|
381
|
-
o.on '-V', '--version', "Print version and exit" do |arg|
|
380
|
+
o.on "-V", "--version", "Print version and exit" do
|
382
381
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
383
382
|
die(0)
|
384
383
|
end
|
385
|
-
|
384
|
+
}
|
386
385
|
|
387
|
-
|
388
|
-
|
389
|
-
logger.info
|
386
|
+
parser.banner = "sidekiq [options]"
|
387
|
+
parser.on_tail "-h", "--help", "Show help" do
|
388
|
+
logger.info parser
|
390
389
|
die 1
|
391
390
|
end
|
392
391
|
|
393
|
-
|
394
|
-
|
395
|
-
opts
|
392
|
+
parser
|
396
393
|
end
|
397
394
|
|
398
395
|
def initialize_logger
|
399
|
-
|
400
|
-
|
401
|
-
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
402
|
-
end
|
403
|
-
|
404
|
-
def write_pid
|
405
|
-
if path = options[:pidfile]
|
406
|
-
pidfile = File.expand_path(path)
|
407
|
-
File.open(pidfile, 'w') do |f|
|
408
|
-
f.puts ::Process.pid
|
409
|
-
end
|
410
|
-
end
|
396
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
411
397
|
end
|
412
398
|
|
413
399
|
def parse_config(path)
|
414
|
-
|
400
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
401
|
+
erb.filename = File.expand_path(path)
|
402
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
415
403
|
|
416
404
|
if opts.respond_to? :deep_symbolize_keys!
|
417
405
|
opts.deep_symbolize_keys!
|
@@ -420,26 +408,16 @@ module Sidekiq
|
|
420
408
|
end
|
421
409
|
|
422
410
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
423
|
-
|
411
|
+
opts.delete(:strict)
|
424
412
|
|
425
|
-
ns = opts.delete(:namespace)
|
426
|
-
if ns
|
427
|
-
# logger hasn't been initialized yet, puts is all we have.
|
428
|
-
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
429
|
-
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
430
|
-
end
|
431
413
|
opts
|
432
414
|
end
|
433
415
|
|
434
|
-
def
|
435
|
-
|
436
|
-
end
|
437
|
-
|
438
|
-
def parse_queue(opts, queue, weight = nil)
|
439
|
-
opts[:queues] ||= []
|
440
|
-
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
441
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue }
|
442
|
-
opts[:strict] = false if weight.to_i > 0
|
416
|
+
def rails_app?
|
417
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
443
418
|
end
|
444
419
|
end
|
445
420
|
end
|
421
|
+
|
422
|
+
require "sidekiq/systemd"
|
423
|
+
require "sidekiq/metrics/tracking"
|