sidekiq 5.2.2 → 7.2.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (149) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +657 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -50
  5. data/bin/sidekiq +22 -3
  6. data/bin/sidekiqload +213 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +558 -353
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +238 -260
  15. data/lib/sidekiq/client.rb +127 -102
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +287 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +49 -42
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +35 -9
  23. data/lib/sidekiq/job_retry.rb +162 -102
  24. data/lib/sidekiq/job_util.rb +107 -0
  25. data/lib/sidekiq/launcher.rb +203 -105
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +43 -46
  28. data/lib/sidekiq/metrics/query.rb +155 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +113 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  33. data/lib/sidekiq/middleware/i18n.rb +7 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +28 -16
  37. data/lib/sidekiq/processor.rb +127 -118
  38. data/lib/sidekiq/rails.rb +50 -39
  39. data/lib/sidekiq/redis_client_adapter.rb +111 -0
  40. data/lib/sidekiq/redis_connection.rb +40 -89
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +111 -49
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +6 -5
  46. data/lib/sidekiq/testing.rb +90 -89
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +3 -1
  49. data/lib/sidekiq/web/action.rb +15 -11
  50. data/lib/sidekiq/web/application.rb +189 -79
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +160 -114
  53. data/lib/sidekiq/web/router.rb +23 -19
  54. data/lib/sidekiq/web.rb +68 -107
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +94 -182
  57. data/sidekiq.gemspec +25 -18
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +146 -61
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +182 -0
  64. data/web/assets/javascripts/dashboard.js +35 -283
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +2 -95
  68. data/web/assets/stylesheets/application.css +143 -521
  69. data/web/assets/stylesheets/bootstrap.css +1 -1
  70. data/web/locales/ar.yml +71 -65
  71. data/web/locales/cs.yml +62 -62
  72. data/web/locales/da.yml +60 -53
  73. data/web/locales/de.yml +65 -53
  74. data/web/locales/el.yml +43 -24
  75. data/web/locales/en.yml +86 -66
  76. data/web/locales/es.yml +70 -54
  77. data/web/locales/fa.yml +65 -65
  78. data/web/locales/fr.yml +83 -62
  79. data/web/locales/gd.yml +99 -0
  80. data/web/locales/he.yml +65 -64
  81. data/web/locales/hi.yml +59 -59
  82. data/web/locales/it.yml +53 -53
  83. data/web/locales/ja.yml +75 -64
  84. data/web/locales/ko.yml +52 -52
  85. data/web/locales/lt.yml +83 -0
  86. data/web/locales/nb.yml +61 -61
  87. data/web/locales/nl.yml +52 -52
  88. data/web/locales/pl.yml +45 -45
  89. data/web/locales/pt-br.yml +83 -55
  90. data/web/locales/pt.yml +51 -51
  91. data/web/locales/ru.yml +68 -63
  92. data/web/locales/sv.yml +53 -53
  93. data/web/locales/ta.yml +60 -60
  94. data/web/locales/uk.yml +62 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +83 -0
  97. data/web/locales/zh-cn.yml +43 -16
  98. data/web/locales/zh-tw.yml +42 -8
  99. data/web/views/_footer.erb +6 -3
  100. data/web/views/_job_info.erb +21 -4
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +4 -18
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +3 -6
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +77 -27
  107. data/web/views/dashboard.erb +48 -18
  108. data/web/views/dead.erb +3 -3
  109. data/web/views/filtering.erb +7 -0
  110. data/web/views/layout.erb +3 -1
  111. data/web/views/metrics.erb +91 -0
  112. data/web/views/metrics_for_job.erb +59 -0
  113. data/web/views/morgue.erb +14 -15
  114. data/web/views/queue.erb +33 -24
  115. data/web/views/queues.erb +19 -5
  116. data/web/views/retries.erb +16 -17
  117. data/web/views/retry.erb +3 -3
  118. data/web/views/scheduled.erb +17 -15
  119. metadata +80 -65
  120. data/.github/contributing.md +0 -32
  121. data/.github/issue_template.md +0 -11
  122. data/.gitignore +0 -13
  123. data/.travis.yml +0 -14
  124. data/3.0-Upgrade.md +0 -70
  125. data/4.0-Upgrade.md +0 -53
  126. data/5.0-Upgrade.md +0 -56
  127. data/COMM-LICENSE +0 -95
  128. data/Ent-Changes.md +0 -221
  129. data/Gemfile +0 -14
  130. data/LICENSE +0 -9
  131. data/Pro-2.0-Upgrade.md +0 -138
  132. data/Pro-3.0-Upgrade.md +0 -44
  133. data/Pro-4.0-Upgrade.md +0 -35
  134. data/Pro-Changes.md +0 -746
  135. data/Rakefile +0 -8
  136. data/bin/sidekiqctl +0 -99
  137. data/code_of_conduct.md +0 -50
  138. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  139. data/lib/sidekiq/core_ext.rb +0 -1
  140. data/lib/sidekiq/delay.rb +0 -42
  141. data/lib/sidekiq/exception_handler.rb +0 -29
  142. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  143. data/lib/sidekiq/extensions/active_record.rb +0 -40
  144. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  145. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  146. data/lib/sidekiq/logging.rb +0 -122
  147. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  148. data/lib/sidekiq/util.rb +0 -66
  149. data/lib/sidekiq/worker.rb +0 -204
data/lib/sidekiq/cli.rb CHANGED
@@ -1,45 +1,34 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  $stdout.sync = true
3
4
 
4
- require 'yaml'
5
- require 'singleton'
6
- require 'optparse'
7
- require 'erb'
8
- require 'fileutils'
5
+ require "yaml"
6
+ require "singleton"
7
+ require "optparse"
8
+ require "erb"
9
+ require "fileutils"
9
10
 
10
- require 'sidekiq'
11
- require 'sidekiq/util'
11
+ require "sidekiq"
12
+ require "sidekiq/config"
13
+ require "sidekiq/component"
14
+ require "sidekiq/capsule"
15
+ require "sidekiq/launcher"
12
16
 
13
- module Sidekiq
17
+ module Sidekiq # :nodoc:
14
18
  class CLI
15
- include Util
19
+ include Sidekiq::Component
16
20
  include Singleton unless $TESTING
17
21
 
18
- PROCTITLES = [
19
- proc { 'sidekiq' },
20
- proc { Sidekiq::VERSION },
21
- proc { |me, data| data['tag'] },
22
- proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
23
- proc { |me, data| "stopping" if me.stopping? },
24
- ]
25
-
26
- # Used for CLI testing
27
- attr_accessor :code
28
22
  attr_accessor :launcher
29
23
  attr_accessor :environment
24
+ attr_accessor :config
30
25
 
31
- def initialize
32
- @code = nil
33
- end
34
-
35
- def parse(args=ARGV)
36
- @code = nil
26
+ def parse(args = ARGV.dup)
27
+ @config ||= Sidekiq.default_configuration
37
28
 
38
29
  setup_options(args)
39
30
  initialize_logger
40
31
  validate!
41
- daemonize
42
- write_pid
43
32
  end
44
33
 
45
34
  def jruby?
@@ -49,189 +38,206 @@ module Sidekiq
49
38
  # Code within this method is not tested because it alters
50
39
  # global process state irreversibly. PRs which improve the
51
40
  # test coverage of Sidekiq::CLI are welcomed.
52
- def run
53
- boot_system
54
- print_banner
41
+ def run(boot_app: true)
42
+ boot_application if boot_app
55
43
 
56
- self_read, self_write = IO.pipe
57
- sigs = %w(INT TERM TTIN TSTP)
58
- # USR1 and USR2 don't work on the JVM
59
- if !jruby?
60
- sigs << 'USR1'
61
- sigs << 'USR2'
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
45
+ print_banner
62
46
  end
47
+ logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
63
48
 
49
+ self_read, self_write = IO.pipe
50
+ sigs = %w[INT TERM TTIN TSTP]
51
+ # USR1 and USR2 don't work on the JVM
52
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
64
53
  sigs.each do |sig|
65
- begin
66
- trap sig do
67
- self_write.write("#{sig}\n")
54
+ old_handler = Signal.trap(sig) do
55
+ if old_handler.respond_to?(:call)
56
+ begin
57
+ old_handler.call
58
+ rescue Exception => exc
59
+ # signal handlers can't use Logger so puts only
60
+ puts ["Error in #{sig} handler", exc].inspect
61
+ end
68
62
  end
69
- rescue ArgumentError
70
- puts "Signal #{sig} not supported"
63
+ self_write.puts(sig)
71
64
  end
65
+ rescue ArgumentError
66
+ puts "Signal #{sig} not supported"
72
67
  end
73
68
 
74
69
  logger.info "Running in #{RUBY_DESCRIPTION}"
75
70
  logger.info Sidekiq::LICENSE
76
- logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
71
+ logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
77
72
 
78
73
  # touch the connection pool so it is created before we
79
74
  # fire startup and start multithreading.
80
- ver = Sidekiq.redis_info['redis_version']
81
- raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
75
+ info = @config.redis_info
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
78
+
79
+ maxmemory_policy = info["maxmemory_policy"]
80
+ if maxmemory_policy != "noeviction" && maxmemory_policy != ""
81
+ # Redis Enterprise Cloud returns "" for their policy 😳
82
+ logger.warn <<~EOM
83
+
84
+
85
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
86
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
87
+ See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
88
+
89
+ EOM
90
+ end
82
91
 
83
92
  # Since the user can pass us a connection pool explicitly in the initializer, we
84
93
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
85
- cursize = Sidekiq.redis_pool.size
86
- needed = Sidekiq.options[:concurrency] + 2
87
- raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
94
+ @config.capsules.each_pair do |name, cap|
95
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
96
+ end
88
97
 
89
98
  # cache process identity
90
- Sidekiq.options[:identity] = identity
99
+ @config[:identity] = identity
91
100
 
92
101
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
93
- Sidekiq.server_middleware
102
+ @config.server_middleware
94
103
 
95
104
  # Before this point, the process is initializing with just the main thread.
96
105
  # Starting here the process will now have multiple threads running.
97
106
  fire_event(:startup, reverse: false, reraise: true)
98
107
 
99
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
100
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
108
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
109
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
110
+
111
+ launch(self_read)
112
+ end
101
113
 
102
- if !options[:daemon]
103
- logger.info 'Starting processing, hit Ctrl-C to stop'
114
+ def launch(self_read)
115
+ if environment == "development" && $stdout.tty?
116
+ logger.info "Starting processing, hit Ctrl-C to stop"
104
117
  end
105
118
 
106
- require 'sidekiq/launcher'
107
- @launcher = Sidekiq::Launcher.new(options)
119
+ @launcher = Sidekiq::Launcher.new(@config)
108
120
 
109
121
  begin
110
122
  launcher.run
111
123
 
112
- while readable_io = IO.select([self_read])
113
- signal = readable_io.first[0].gets.strip
124
+ while self_read.wait_readable
125
+ signal = self_read.gets.strip
114
126
  handle_signal(signal)
115
127
  end
116
128
  rescue Interrupt
117
- logger.info 'Shutting down'
129
+ logger.info "Shutting down"
118
130
  launcher.stop
119
- # Explicitly exit so busy Processor threads can't block
120
- # process shutdown.
121
131
  logger.info "Bye!"
132
+
133
+ # Explicitly exit so busy Processor threads won't block process shutdown.
134
+ #
135
+ # NB: slow at_exit handlers will prevent a timely exit if they take
136
+ # a while to run. If Sidekiq is getting here but the process isn't exiting,
137
+ # use the TTIN signal to determine where things are stuck.
122
138
  exit(0)
123
139
  end
124
140
  end
125
141
 
142
+ HOLIDAY_COLORS = {
143
+ # got other color-specific holidays from around the world?
144
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
145
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
146
+ "10-31" => "\e[38;5;208m" # Halloween orange
147
+ }
148
+
149
+ def self.day
150
+ @@day ||= begin
151
+ t = Date.today
152
+ "#{t.month}-#{t.day}"
153
+ end
154
+ end
155
+
156
+ def self.r
157
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
158
+ end
159
+
160
+ def self.b
161
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
162
+ end
163
+
164
+ def self.w
165
+ "\e[1;37m"
166
+ end
167
+
168
+ def self.reset
169
+ @@b = @@r = @@day = nil
170
+ "\e[0m"
171
+ end
172
+
126
173
  def self.banner
127
- %q{
128
- m,
129
- `$b
130
- .ss, $$: .,d$
131
- `$$P,d$P' .,md$P"'
132
- ,$$$$$bmmd$$$P^'
133
- .d$$$$$$$$$$P'
134
- $$^' `"^$$$' ____ _ _ _ _
135
- $: ,$$: / ___|(_) __| | ___| | _(_) __ _
136
- `b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
137
- $$: ___) | | (_| | __/ <| | (_| |
138
- $$ |____/|_|\__,_|\___|_|\_\_|\__, |
139
- .d$$ |_|
140
- }
174
+ %{
175
+ #{w} m,
176
+ #{w} `$b
177
+ #{w} .ss, $$: .,d$
178
+ #{w} `$$P,d$P' .,md$P"'
179
+ #{w} ,$$$$$b#{b}/#{w}md$$$P^'
180
+ #{w} .d$$$$$$#{b}/#{w}$$$P'
181
+ #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
182
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
183
+ #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
184
+ #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
185
+ #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
186
+ #{w} .d$$ #{r} |_|
187
+ #{reset}}
141
188
  end
142
189
 
143
190
  SIGNAL_HANDLERS = {
144
191
  # Ctrl-C in terminal
145
- 'INT' => ->(cli) { raise Interrupt },
192
+ "INT" => ->(cli) { raise Interrupt },
146
193
  # TERM is the signal that Sidekiq must exit.
147
194
  # Heroku sends TERM and then waits 30 seconds for process to exit.
148
- 'TERM' => ->(cli) { raise Interrupt },
149
- 'USR1' => ->(cli) {
150
- Sidekiq.logger.info "Received USR1, no longer accepting new work"
195
+ "TERM" => ->(cli) { raise Interrupt },
196
+ "TSTP" => ->(cli) {
197
+ cli.logger.info "Received TSTP, no longer accepting new work"
151
198
  cli.launcher.quiet
152
199
  },
153
- 'TSTP' => ->(cli) {
154
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
155
- cli.launcher.quiet
156
- },
157
- 'USR2' => ->(cli) {
158
- if Sidekiq.options[:logfile]
159
- Sidekiq.logger.info "Received USR2, reopening log file"
160
- Sidekiq::Logging.reopen_logs
161
- end
162
- },
163
- 'TTIN' => ->(cli) {
200
+ "TTIN" => ->(cli) {
164
201
  Thread.list.each do |thread|
165
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
202
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
166
203
  if thread.backtrace
167
- Sidekiq.logger.warn thread.backtrace.join("\n")
204
+ cli.logger.warn thread.backtrace.join("\n")
168
205
  else
169
- Sidekiq.logger.warn "<no backtrace available>"
206
+ cli.logger.warn "<no backtrace available>"
170
207
  end
171
208
  end
172
- },
209
+ }
173
210
  }
211
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
212
+ SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
174
213
 
175
214
  def handle_signal(sig)
176
- Sidekiq.logger.debug "Got #{sig} signal"
177
- handy = SIGNAL_HANDLERS[sig]
178
- if handy
179
- handy.call(self)
180
- else
181
- Sidekiq.logger.info { "No signal handler for #{sig}" }
182
- end
215
+ logger.debug "Got #{sig} signal"
216
+ SIGNAL_HANDLERS[sig].call(self)
183
217
  end
184
218
 
185
219
  private
186
220
 
187
221
  def print_banner
188
- # Print logo and banner for development
189
- if environment == 'development' && $stdout.tty?
190
- puts "\e[#{31}m"
191
- puts Sidekiq::CLI.banner
192
- puts "\e[0m"
193
- end
194
- end
195
-
196
- def daemonize
197
- return unless options[:daemon]
198
-
199
- raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
200
- files_to_reopen = []
201
- ObjectSpace.each_object(File) do |file|
202
- files_to_reopen << file unless file.closed?
203
- end
204
-
205
- ::Process.daemon(true, true)
206
-
207
- files_to_reopen.each do |file|
208
- begin
209
- file.reopen file.path, "a+"
210
- file.sync = true
211
- rescue ::Exception
212
- end
213
- end
214
-
215
- [$stdout, $stderr].each do |io|
216
- File.open(options[:logfile], 'ab') do |f|
217
- io.reopen(f)
218
- end
219
- io.sync = true
220
- end
221
- $stdin.reopen('/dev/null')
222
-
223
- initialize_logger
222
+ puts "\e[31m"
223
+ puts Sidekiq::CLI.banner
224
+ puts "\e[0m"
224
225
  end
225
226
 
226
227
  def set_environment(cli_env)
227
- @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
228
+ # See #984 for discussion.
229
+ # APP_ENV is now the preferred ENV term since it is not tech-specific.
230
+ # Both Sinatra 2.0+ and Sidekiq support this term.
231
+ # RAILS_ENV and RACK_ENV are there for legacy support.
232
+ @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
233
+ config[:environment] = @environment
228
234
  end
229
235
 
230
236
  def symbolize_keys_deep!(hash)
231
237
  hash.keys.each do |k|
232
238
  symkey = k.respond_to?(:to_sym) ? k.to_sym : k
233
239
  hash[symkey] = hash.delete k
234
- symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
240
+ symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
235
241
  end
236
242
  end
237
243
 
@@ -239,58 +245,74 @@ module Sidekiq
239
245
  alias_method :☠, :exit
240
246
 
241
247
  def setup_options(args)
248
+ # parse CLI options
242
249
  opts = parse_options(args)
250
+
243
251
  set_environment opts[:environment]
244
252
 
245
- cfile = opts[:config_file]
246
- opts = parse_config(cfile).merge(opts) if cfile
253
+ # check config file presence
254
+ if opts[:config_file]
255
+ unless File.exist?(opts[:config_file])
256
+ raise ArgumentError, "No such file #{opts[:config_file]}"
257
+ end
258
+ else
259
+ config_dir = if File.directory?(opts[:require].to_s)
260
+ File.join(opts[:require], "config")
261
+ else
262
+ File.join(@config[:require], "config")
263
+ end
247
264
 
248
- opts[:strict] = true if opts[:strict].nil?
249
- opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
265
+ %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
266
+ path = File.join(config_dir, config_file)
267
+ opts[:config_file] ||= path if File.exist?(path)
268
+ end
269
+ end
250
270
 
251
- options.merge!(opts)
252
- end
271
+ # parse config file options
272
+ opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
273
+
274
+ # set defaults
275
+ opts[:queues] = ["default"] if opts[:queues].nil?
276
+ opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
277
+
278
+ # merge with defaults
279
+ @config.merge!(opts)
253
280
 
254
- def options
255
- Sidekiq.options
281
+ @config.default_capsule.tap do |cap|
282
+ cap.queues = opts[:queues]
283
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
284
+ end
285
+
286
+ opts[:capsules]&.each do |name, cap_config|
287
+ @config.capsule(name.to_s) do |cap|
288
+ cap.queues = cap_config[:queues]
289
+ cap.concurrency = cap_config[:concurrency]
290
+ end
291
+ end
256
292
  end
257
293
 
258
- def boot_system
259
- ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
260
-
261
- raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
262
-
263
- if File.directory?(options[:require])
264
- require 'rails'
265
- if ::Rails::VERSION::MAJOR < 4
266
- raise "Sidekiq no longer supports this version of Rails"
267
- elsif ::Rails::VERSION::MAJOR == 4
268
- # Painful contortions, see 1791 for discussion
269
- # No autoloading, we want to force eager load for everything.
270
- require File.expand_path("#{options[:require]}/config/application.rb")
271
- ::Rails::Application.initializer "sidekiq.eager_load" do
272
- ::Rails.application.config.eager_load = true
273
- end
274
- require 'sidekiq/rails'
275
- require File.expand_path("#{options[:require]}/config/environment.rb")
276
- else
277
- require 'sidekiq/rails'
278
- require File.expand_path("#{options[:require]}/config/environment.rb")
294
+ def boot_application
295
+ ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
296
+
297
+ if File.directory?(@config[:require])
298
+ require "rails"
299
+ if ::Rails::VERSION::MAJOR < 6
300
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
279
301
  end
280
- options[:tag] ||= default_tag
302
+ require "sidekiq/rails"
303
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
304
+ @config[:tag] ||= default_tag
281
305
  else
282
- not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
283
- "./#{options[:require]} or /path/to/#{options[:require]}"
284
-
285
- require(options[:require]) || raise(ArgumentError, not_required_message)
306
+ require @config[:require]
286
307
  end
287
308
  end
288
309
 
289
310
  def default_tag
290
311
  dir = ::Rails.root
291
312
  name = File.basename(dir)
292
- if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
293
- if File.basename(prevdir) == 'releases'
313
+ prevdir = File.dirname(dir) # Capistrano release directory?
314
+ if name.to_i != 0 && prevdir
315
+ if File.basename(prevdir) == "releases"
294
316
  return File.basename(File.dirname(prevdir))
295
317
  end
296
318
  end
@@ -298,59 +320,52 @@ module Sidekiq
298
320
  end
299
321
 
300
322
  def validate!
301
- options[:queues] << 'default' if options[:queues].empty?
302
-
303
- if !File.exist?(options[:require]) ||
304
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
323
+ if !File.exist?(@config[:require]) ||
324
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
305
325
  logger.info "=================================================================="
306
- logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
307
- logger.info " to load your worker classes with -r [DIR|FILE]."
326
+ logger.info " Please point Sidekiq to a Rails application or a Ruby file "
327
+ logger.info " to load your job classes with -r [DIR|FILE]."
308
328
  logger.info "=================================================================="
309
329
  logger.info @parser
310
330
  die(1)
311
331
  end
312
332
 
313
333
  [:concurrency, :timeout].each do |opt|
314
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
334
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
315
335
  end
316
336
  end
317
337
 
318
338
  def parse_options(argv)
319
339
  opts = {}
340
+ @parser = option_parser(opts)
341
+ @parser.parse!(argv)
342
+ opts
343
+ end
320
344
 
321
- @parser = OptionParser.new do |o|
322
- o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
345
+ def option_parser(opts)
346
+ parser = OptionParser.new { |o|
347
+ o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
323
348
  opts[:concurrency] = Integer(arg)
324
349
  end
325
350
 
326
- o.on '-d', '--daemon', "Daemonize process" do |arg|
327
- opts[:daemon] = arg
328
- end
329
-
330
- o.on '-e', '--environment ENV', "Application environment" do |arg|
351
+ o.on "-e", "--environment ENV", "Application environment" do |arg|
331
352
  opts[:environment] = arg
332
353
  end
333
354
 
334
- o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
355
+ o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
335
356
  opts[:tag] = arg
336
357
  end
337
358
 
338
- # this index remains here for backwards compatibility but none of the Sidekiq
339
- # family use this value anymore. it was used by Pro's original reliable_fetch.
340
- o.on '-i', '--index INT', "unique process index on this machine" do |arg|
341
- opts[:index] = Integer(arg.match(/\d+/)[0])
342
- end
343
-
344
359
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
345
- queue, weight = arg.split(",")
346
- parse_queue opts, queue, weight
360
+ opts[:queues] ||= []
361
+ opts[:queues] << arg
347
362
  end
348
363
 
349
- o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
364
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
350
365
  opts[:require] = arg
351
366
  end
352
367
 
353
- o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
368
+ o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
354
369
  opts[:timeout] = Integer(arg)
355
370
  end
356
371
 
@@ -358,88 +373,51 @@ module Sidekiq
358
373
  opts[:verbose] = arg
359
374
  end
360
375
 
361
- o.on '-C', '--config PATH', "path to YAML config file" do |arg|
376
+ o.on "-C", "--config PATH", "path to YAML config file" do |arg|
362
377
  opts[:config_file] = arg
363
378
  end
364
379
 
365
- o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
366
- opts[:logfile] = arg
367
- end
368
-
369
- o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
370
- opts[:pidfile] = arg
371
- end
372
-
373
- o.on '-V', '--version', "Print version and exit" do |arg|
380
+ o.on "-V", "--version", "Print version and exit" do
374
381
  puts "Sidekiq #{Sidekiq::VERSION}"
375
382
  die(0)
376
383
  end
377
- end
384
+ }
378
385
 
379
- @parser.banner = "sidekiq [options]"
380
- @parser.on_tail "-h", "--help", "Show help" do
381
- logger.info @parser
386
+ parser.banner = "sidekiq [options]"
387
+ parser.on_tail "-h", "--help", "Show help" do
388
+ logger.info parser
382
389
  die 1
383
390
  end
384
- @parser.parse!(argv)
385
391
 
386
- %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
387
- opts[:config_file] ||= filename if File.exist?(filename)
388
- end
389
-
390
- opts
392
+ parser
391
393
  end
392
394
 
393
395
  def initialize_logger
394
- Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
395
-
396
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
396
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
397
397
  end
398
398
 
399
- def write_pid
400
- if path = options[:pidfile]
401
- pidfile = File.expand_path(path)
402
- File.open(pidfile, 'w') do |f|
403
- f.puts ::Process.pid
404
- end
405
- end
406
- end
407
-
408
- def parse_config(cfile)
409
- opts = {}
410
- if File.exist?(cfile)
411
- opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
412
-
413
- if opts.respond_to? :deep_symbolize_keys!
414
- opts.deep_symbolize_keys!
415
- else
416
- symbolize_keys_deep!(opts)
417
- end
399
+ def parse_config(path)
400
+ erb = ERB.new(File.read(path), trim_mode: "-")
401
+ erb.filename = File.expand_path(path)
402
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
418
403
 
419
- opts = opts.merge(opts.delete(environment.to_sym) || {})
420
- parse_queues(opts, opts.delete(:queues) || [])
404
+ if opts.respond_to? :deep_symbolize_keys!
405
+ opts.deep_symbolize_keys!
421
406
  else
422
- # allow a non-existent config file so Sidekiq
423
- # can be deployed by cap with just the defaults.
407
+ symbolize_keys_deep!(opts)
424
408
  end
425
- ns = opts.delete(:namespace)
426
- if ns
427
- # logger hasn't been initialized yet, puts is all we have.
428
- puts("namespace should be set in your ruby initializer, is ignored in config file")
429
- puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
430
- end
431
- opts
432
- end
433
409
 
434
- def parse_queues(opts, queues_and_weights)
435
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
410
+ opts = opts.merge(opts.delete(environment.to_sym) || {})
411
+ opts.delete(:strict)
412
+
413
+ opts
436
414
  end
437
415
 
438
- def parse_queue(opts, q, weight=nil)
439
- opts[:queues] ||= []
440
- raise ArgumentError, "queues: #{q} cannot be defined twice" if opts[:queues].include?(q)
441
- [weight.to_i, 1].max.times { opts[:queues] << q }
442
- opts[:strict] = false if weight.to_i > 0
416
+ def rails_app?
417
+ defined?(::Rails) && ::Rails.respond_to?(:application)
443
418
  end
444
419
  end
445
420
  end
421
+
422
+ require "sidekiq/systemd"
423
+ require "sidekiq/metrics/tracking"