sidekiq 5.1.1 → 7.1.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (149) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +627 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -50
  5. data/bin/sidekiq +22 -3
  6. data/bin/sidekiqload +213 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +566 -329
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +241 -256
  15. data/lib/sidekiq/client.rb +125 -102
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +278 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +49 -42
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +36 -9
  23. data/lib/sidekiq/job_retry.rb +147 -98
  24. data/lib/sidekiq/job_util.rb +105 -0
  25. data/lib/sidekiq/launcher.rb +207 -103
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +43 -47
  28. data/lib/sidekiq/metrics/query.rb +153 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +113 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  33. data/lib/sidekiq/middleware/i18n.rb +7 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +28 -16
  37. data/lib/sidekiq/processor.rb +159 -107
  38. data/lib/sidekiq/rails.rb +54 -43
  39. data/lib/sidekiq/redis_client_adapter.rb +96 -0
  40. data/lib/sidekiq/redis_connection.rb +39 -81
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +139 -48
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +6 -5
  46. data/lib/sidekiq/testing.rb +70 -88
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +3 -1
  49. data/lib/sidekiq/web/action.rb +15 -11
  50. data/lib/sidekiq/web/application.rb +143 -77
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +144 -106
  53. data/lib/sidekiq/web/router.rb +23 -19
  54. data/lib/sidekiq/web.rb +60 -111
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +94 -183
  57. data/sidekiq.gemspec +25 -23
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +130 -61
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +166 -0
  64. data/web/assets/javascripts/dashboard.js +36 -282
  65. data/web/assets/javascripts/metrics.js +264 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +2 -95
  68. data/web/assets/stylesheets/application.css +134 -521
  69. data/web/assets/stylesheets/bootstrap.css +2 -2
  70. data/web/locales/ar.yml +71 -64
  71. data/web/locales/cs.yml +62 -62
  72. data/web/locales/da.yml +60 -53
  73. data/web/locales/de.yml +65 -53
  74. data/web/locales/el.yml +43 -24
  75. data/web/locales/en.yml +84 -65
  76. data/web/locales/es.yml +70 -54
  77. data/web/locales/fa.yml +65 -65
  78. data/web/locales/fr.yml +83 -62
  79. data/web/locales/gd.yml +99 -0
  80. data/web/locales/he.yml +65 -64
  81. data/web/locales/hi.yml +59 -59
  82. data/web/locales/it.yml +53 -53
  83. data/web/locales/ja.yml +75 -64
  84. data/web/locales/ko.yml +52 -52
  85. data/web/locales/lt.yml +83 -0
  86. data/web/locales/nb.yml +61 -61
  87. data/web/locales/nl.yml +52 -52
  88. data/web/locales/pl.yml +45 -45
  89. data/web/locales/pt-br.yml +63 -55
  90. data/web/locales/pt.yml +51 -51
  91. data/web/locales/ru.yml +68 -63
  92. data/web/locales/sv.yml +53 -53
  93. data/web/locales/ta.yml +60 -60
  94. data/web/locales/uk.yml +62 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +83 -0
  97. data/web/locales/zh-cn.yml +43 -16
  98. data/web/locales/zh-tw.yml +42 -8
  99. data/web/views/_footer.erb +8 -2
  100. data/web/views/_job_info.erb +21 -4
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +4 -18
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +3 -6
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +75 -25
  107. data/web/views/dashboard.erb +58 -18
  108. data/web/views/dead.erb +3 -3
  109. data/web/views/layout.erb +4 -2
  110. data/web/views/metrics.erb +82 -0
  111. data/web/views/metrics_for_job.erb +68 -0
  112. data/web/views/morgue.erb +14 -15
  113. data/web/views/queue.erb +33 -23
  114. data/web/views/queues.erb +14 -4
  115. data/web/views/retries.erb +19 -16
  116. data/web/views/retry.erb +3 -3
  117. data/web/views/scheduled.erb +17 -15
  118. metadata +71 -140
  119. data/.github/contributing.md +0 -32
  120. data/.github/issue_template.md +0 -11
  121. data/.gitignore +0 -13
  122. data/.travis.yml +0 -14
  123. data/3.0-Upgrade.md +0 -70
  124. data/4.0-Upgrade.md +0 -53
  125. data/5.0-Upgrade.md +0 -56
  126. data/COMM-LICENSE +0 -95
  127. data/Ent-Changes.md +0 -210
  128. data/Gemfile +0 -8
  129. data/LICENSE +0 -9
  130. data/Pro-2.0-Upgrade.md +0 -138
  131. data/Pro-3.0-Upgrade.md +0 -44
  132. data/Pro-4.0-Upgrade.md +0 -35
  133. data/Pro-Changes.md +0 -716
  134. data/Rakefile +0 -8
  135. data/bin/sidekiqctl +0 -99
  136. data/code_of_conduct.md +0 -50
  137. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  138. data/lib/sidekiq/core_ext.rb +0 -1
  139. data/lib/sidekiq/delay.rb +0 -41
  140. data/lib/sidekiq/exception_handler.rb +0 -29
  141. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  142. data/lib/sidekiq/extensions/active_record.rb +0 -40
  143. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  144. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  145. data/lib/sidekiq/logging.rb +0 -122
  146. data/lib/sidekiq/middleware/server/active_record.rb +0 -22
  147. data/lib/sidekiq/middleware/server/active_record_cache.rb +0 -11
  148. data/lib/sidekiq/util.rb +0 -66
  149. data/lib/sidekiq/worker.rb +0 -204
data/lib/sidekiq/cli.rb CHANGED
@@ -1,46 +1,34 @@
1
- # encoding: utf-8
2
1
  # frozen_string_literal: true
2
+
3
3
  $stdout.sync = true
4
4
 
5
- require 'yaml'
6
- require 'singleton'
7
- require 'optparse'
8
- require 'erb'
9
- require 'fileutils'
5
+ require "yaml"
6
+ require "singleton"
7
+ require "optparse"
8
+ require "erb"
9
+ require "fileutils"
10
10
 
11
- require 'sidekiq'
12
- require 'sidekiq/util'
11
+ require "sidekiq"
12
+ require "sidekiq/config"
13
+ require "sidekiq/component"
14
+ require "sidekiq/capsule"
15
+ require "sidekiq/launcher"
13
16
 
14
- module Sidekiq
17
+ module Sidekiq # :nodoc:
15
18
  class CLI
16
- include Util
19
+ include Sidekiq::Component
17
20
  include Singleton unless $TESTING
18
21
 
19
- PROCTITLES = [
20
- proc { 'sidekiq'.freeze },
21
- proc { Sidekiq::VERSION },
22
- proc { |me, data| data['tag'] },
23
- proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
24
- proc { |me, data| "stopping" if me.stopping? },
25
- ]
26
-
27
- # Used for CLI testing
28
- attr_accessor :code
29
22
  attr_accessor :launcher
30
23
  attr_accessor :environment
24
+ attr_accessor :config
31
25
 
32
- def initialize
33
- @code = nil
34
- end
35
-
36
- def parse(args=ARGV)
37
- @code = nil
26
+ def parse(args = ARGV.dup)
27
+ @config ||= Sidekiq.default_configuration
38
28
 
39
29
  setup_options(args)
40
30
  initialize_logger
41
31
  validate!
42
- daemonize
43
- write_pid
44
32
  end
45
33
 
46
34
  def jruby?
@@ -50,183 +38,206 @@ module Sidekiq
50
38
  # Code within this method is not tested because it alters
51
39
  # global process state irreversibly. PRs which improve the
52
40
  # test coverage of Sidekiq::CLI are welcomed.
53
- def run
54
- boot_system
55
- print_banner
41
+ def run(boot_app: true)
42
+ boot_application if boot_app
56
43
 
57
- self_read, self_write = IO.pipe
58
- sigs = %w(INT TERM TTIN TSTP)
59
- # USR1 and USR2 don't work on the JVM
60
- if !jruby?
61
- sigs << 'USR1'
62
- sigs << 'USR2'
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
45
+ print_banner
63
46
  end
47
+ logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
64
48
 
49
+ self_read, self_write = IO.pipe
50
+ sigs = %w[INT TERM TTIN TSTP]
51
+ # USR1 and USR2 don't work on the JVM
52
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
65
53
  sigs.each do |sig|
66
- begin
67
- trap sig do
68
- self_write.puts(sig)
54
+ old_handler = Signal.trap(sig) do
55
+ if old_handler.respond_to?(:call)
56
+ begin
57
+ old_handler.call
58
+ rescue Exception => exc
59
+ # signal handlers can't use Logger so puts only
60
+ puts ["Error in #{sig} handler", exc].inspect
61
+ end
69
62
  end
70
- rescue ArgumentError
71
- puts "Signal #{sig} not supported"
63
+ self_write.puts(sig)
72
64
  end
65
+ rescue ArgumentError
66
+ puts "Signal #{sig} not supported"
73
67
  end
74
68
 
75
69
  logger.info "Running in #{RUBY_DESCRIPTION}"
76
70
  logger.info Sidekiq::LICENSE
77
- logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
71
+ logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
78
72
 
79
73
  # touch the connection pool so it is created before we
80
74
  # fire startup and start multithreading.
81
- ver = Sidekiq.redis_info['redis_version']
82
- raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
75
+ info = @config.redis_info
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
78
+
79
+ maxmemory_policy = info["maxmemory_policy"]
80
+ if maxmemory_policy != "noeviction" && maxmemory_policy != ""
81
+ # Redis Enterprise Cloud returns "" for their policy 😳
82
+ logger.warn <<~EOM
83
+
84
+
85
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
86
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
87
+ See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
88
+
89
+ EOM
90
+ end
91
+
92
+ # Since the user can pass us a connection pool explicitly in the initializer, we
93
+ # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
94
+ @config.capsules.each_pair do |name, cap|
95
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
96
+ end
83
97
 
84
98
  # cache process identity
85
- Sidekiq.options[:identity] = identity
99
+ @config[:identity] = identity
86
100
 
87
101
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
88
- Sidekiq.server_middleware
102
+ @config.server_middleware
89
103
 
90
104
  # Before this point, the process is initializing with just the main thread.
91
105
  # Starting here the process will now have multiple threads running.
92
106
  fire_event(:startup, reverse: false, reraise: true)
93
107
 
94
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
95
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
108
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
109
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
110
+
111
+ launch(self_read)
112
+ end
96
113
 
97
- if !options[:daemon]
98
- logger.info 'Starting processing, hit Ctrl-C to stop'
114
+ def launch(self_read)
115
+ if environment == "development" && $stdout.tty?
116
+ logger.info "Starting processing, hit Ctrl-C to stop"
99
117
  end
100
118
 
101
- require 'sidekiq/launcher'
102
- @launcher = Sidekiq::Launcher.new(options)
119
+ @launcher = Sidekiq::Launcher.new(@config)
103
120
 
104
121
  begin
105
122
  launcher.run
106
123
 
107
- while readable_io = IO.select([self_read])
108
- signal = readable_io.first[0].gets.strip
124
+ while self_read.wait_readable
125
+ signal = self_read.gets.strip
109
126
  handle_signal(signal)
110
127
  end
111
128
  rescue Interrupt
112
- logger.info 'Shutting down'
129
+ logger.info "Shutting down"
113
130
  launcher.stop
114
- # Explicitly exit so busy Processor threads can't block
115
- # process shutdown.
116
131
  logger.info "Bye!"
132
+
133
+ # Explicitly exit so busy Processor threads won't block process shutdown.
134
+ #
135
+ # NB: slow at_exit handlers will prevent a timely exit if they take
136
+ # a while to run. If Sidekiq is getting here but the process isn't exiting,
137
+ # use the TTIN signal to determine where things are stuck.
117
138
  exit(0)
118
139
  end
119
140
  end
120
141
 
142
+ HOLIDAY_COLORS = {
143
+ # got other color-specific holidays from around the world?
144
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
145
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
146
+ "10-31" => "\e[38;5;208m" # Halloween orange
147
+ }
148
+
149
+ def self.day
150
+ @@day ||= begin
151
+ t = Date.today
152
+ "#{t.month}-#{t.day}"
153
+ end
154
+ end
155
+
156
+ def self.r
157
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
158
+ end
159
+
160
+ def self.b
161
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
162
+ end
163
+
164
+ def self.w
165
+ "\e[1;37m"
166
+ end
167
+
168
+ def self.reset
169
+ @@b = @@r = @@day = nil
170
+ "\e[0m"
171
+ end
172
+
121
173
  def self.banner
122
- %q{
123
- m,
124
- `$b
125
- .ss, $$: .,d$
126
- `$$P,d$P' .,md$P"'
127
- ,$$$$$bmmd$$$P^'
128
- .d$$$$$$$$$$P'
129
- $$^' `"^$$$' ____ _ _ _ _
130
- $: ,$$: / ___|(_) __| | ___| | _(_) __ _
131
- `b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
132
- $$: ___) | | (_| | __/ <| | (_| |
133
- $$ |____/|_|\__,_|\___|_|\_\_|\__, |
134
- .d$$ |_|
135
- }
174
+ %{
175
+ #{w} m,
176
+ #{w} `$b
177
+ #{w} .ss, $$: .,d$
178
+ #{w} `$$P,d$P' .,md$P"'
179
+ #{w} ,$$$$$b#{b}/#{w}md$$$P^'
180
+ #{w} .d$$$$$$#{b}/#{w}$$$P'
181
+ #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
182
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
183
+ #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
184
+ #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
185
+ #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
186
+ #{w} .d$$ #{r} |_|
187
+ #{reset}}
136
188
  end
137
189
 
138
190
  SIGNAL_HANDLERS = {
139
191
  # Ctrl-C in terminal
140
- 'INT' => ->(cli) { raise Interrupt },
192
+ "INT" => ->(cli) { raise Interrupt },
141
193
  # TERM is the signal that Sidekiq must exit.
142
194
  # Heroku sends TERM and then waits 30 seconds for process to exit.
143
- 'TERM' => ->(cli) { raise Interrupt },
144
- 'USR1' => ->(cli) {
145
- Sidekiq.logger.info "Received USR1, no longer accepting new work"
146
- cli.launcher.quiet
147
- },
148
- 'TSTP' => ->(cli) {
149
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
195
+ "TERM" => ->(cli) { raise Interrupt },
196
+ "TSTP" => ->(cli) {
197
+ cli.logger.info "Received TSTP, no longer accepting new work"
150
198
  cli.launcher.quiet
151
199
  },
152
- 'USR2' => ->(cli) {
153
- if Sidekiq.options[:logfile]
154
- Sidekiq.logger.info "Received USR2, reopening log file"
155
- Sidekiq::Logging.reopen_logs
156
- end
157
- },
158
- 'TTIN' => ->(cli) {
200
+ "TTIN" => ->(cli) {
159
201
  Thread.list.each do |thread|
160
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
202
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
161
203
  if thread.backtrace
162
- Sidekiq.logger.warn thread.backtrace.join("\n")
204
+ cli.logger.warn thread.backtrace.join("\n")
163
205
  else
164
- Sidekiq.logger.warn "<no backtrace available>"
206
+ cli.logger.warn "<no backtrace available>"
165
207
  end
166
208
  end
167
- },
209
+ }
168
210
  }
211
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
212
+ SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
169
213
 
170
214
  def handle_signal(sig)
171
- Sidekiq.logger.debug "Got #{sig} signal"
172
- handy = SIGNAL_HANDLERS[sig]
173
- if handy
174
- handy.call(self)
175
- else
176
- Sidekiq.logger.info { "No signal handler for #{sig}" }
177
- end
215
+ logger.debug "Got #{sig} signal"
216
+ SIGNAL_HANDLERS[sig].call(self)
178
217
  end
179
218
 
180
219
  private
181
220
 
182
221
  def print_banner
183
- # Print logo and banner for development
184
- if environment == 'development' && $stdout.tty?
185
- puts "\e[#{31}m"
186
- puts Sidekiq::CLI.banner
187
- puts "\e[0m"
188
- end
189
- end
190
-
191
- def daemonize
192
- return unless options[:daemon]
193
-
194
- raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
195
- files_to_reopen = []
196
- ObjectSpace.each_object(File) do |file|
197
- files_to_reopen << file unless file.closed?
198
- end
199
-
200
- ::Process.daemon(true, true)
201
-
202
- files_to_reopen.each do |file|
203
- begin
204
- file.reopen file.path, "a+"
205
- file.sync = true
206
- rescue ::Exception
207
- end
208
- end
209
-
210
- [$stdout, $stderr].each do |io|
211
- File.open(options[:logfile], 'ab') do |f|
212
- io.reopen(f)
213
- end
214
- io.sync = true
215
- end
216
- $stdin.reopen('/dev/null')
217
-
218
- initialize_logger
222
+ puts "\e[31m"
223
+ puts Sidekiq::CLI.banner
224
+ puts "\e[0m"
219
225
  end
220
226
 
221
227
  def set_environment(cli_env)
222
- @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
228
+ # See #984 for discussion.
229
+ # APP_ENV is now the preferred ENV term since it is not tech-specific.
230
+ # Both Sinatra 2.0+ and Sidekiq support this term.
231
+ # RAILS_ENV and RACK_ENV are there for legacy support.
232
+ @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
233
+ config[:environment] = @environment
223
234
  end
224
235
 
225
236
  def symbolize_keys_deep!(hash)
226
237
  hash.keys.each do |k|
227
238
  symkey = k.respond_to?(:to_sym) ? k.to_sym : k
228
239
  hash[symkey] = hash.delete k
229
- symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
240
+ symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
230
241
  end
231
242
  end
232
243
 
@@ -234,58 +245,74 @@ module Sidekiq
234
245
  alias_method :☠, :exit
235
246
 
236
247
  def setup_options(args)
248
+ # parse CLI options
237
249
  opts = parse_options(args)
250
+
238
251
  set_environment opts[:environment]
239
252
 
240
- cfile = opts[:config_file]
241
- opts = parse_config(cfile).merge(opts) if cfile
253
+ # check config file presence
254
+ if opts[:config_file]
255
+ unless File.exist?(opts[:config_file])
256
+ raise ArgumentError, "No such file #{opts[:config_file]}"
257
+ end
258
+ else
259
+ config_dir = if File.directory?(opts[:require].to_s)
260
+ File.join(opts[:require], "config")
261
+ else
262
+ File.join(@config[:require], "config")
263
+ end
242
264
 
243
- opts[:strict] = true if opts[:strict].nil?
244
- opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
265
+ %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
266
+ path = File.join(config_dir, config_file)
267
+ opts[:config_file] ||= path if File.exist?(path)
268
+ end
269
+ end
245
270
 
246
- options.merge!(opts)
247
- end
271
+ # parse config file options
272
+ opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
273
+
274
+ # set defaults
275
+ opts[:queues] = ["default"] if opts[:queues].nil?
276
+ opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
277
+
278
+ # merge with defaults
279
+ @config.merge!(opts)
248
280
 
249
- def options
250
- Sidekiq.options
281
+ @config.default_capsule.tap do |cap|
282
+ cap.queues = opts[:queues]
283
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
284
+ end
285
+
286
+ opts[:capsules]&.each do |name, cap_config|
287
+ @config.capsule(name.to_s) do |cap|
288
+ cap.queues = cap_config[:queues]
289
+ cap.concurrency = cap_config[:concurrency]
290
+ end
291
+ end
251
292
  end
252
293
 
253
- def boot_system
254
- ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
255
-
256
- raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
257
-
258
- if File.directory?(options[:require])
259
- require 'rails'
260
- if ::Rails::VERSION::MAJOR < 4
261
- raise "Sidekiq no longer supports this version of Rails"
262
- elsif ::Rails::VERSION::MAJOR == 4
263
- # Painful contortions, see 1791 for discussion
264
- # No autoloading, we want to force eager load for everything.
265
- require File.expand_path("#{options[:require]}/config/application.rb")
266
- ::Rails::Application.initializer "sidekiq.eager_load" do
267
- ::Rails.application.config.eager_load = true
268
- end
269
- require 'sidekiq/rails'
270
- require File.expand_path("#{options[:require]}/config/environment.rb")
271
- else
272
- require 'sidekiq/rails'
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
294
+ def boot_application
295
+ ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
296
+
297
+ if File.directory?(@config[:require])
298
+ require "rails"
299
+ if ::Rails::VERSION::MAJOR < 6
300
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
274
301
  end
275
- options[:tag] ||= default_tag
302
+ require "sidekiq/rails"
303
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
304
+ @config[:tag] ||= default_tag
276
305
  else
277
- not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
278
- "./#{options[:require]} or /path/to/#{options[:require]}"
279
-
280
- require(options[:require]) || raise(ArgumentError, not_required_message)
306
+ require @config[:require]
281
307
  end
282
308
  end
283
309
 
284
310
  def default_tag
285
311
  dir = ::Rails.root
286
312
  name = File.basename(dir)
287
- if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
288
- if File.basename(prevdir) == 'releases'
313
+ prevdir = File.dirname(dir) # Capistrano release directory?
314
+ if name.to_i != 0 && prevdir
315
+ if File.basename(prevdir) == "releases"
289
316
  return File.basename(File.dirname(prevdir))
290
317
  end
291
318
  end
@@ -293,57 +320,52 @@ module Sidekiq
293
320
  end
294
321
 
295
322
  def validate!
296
- options[:queues] << 'default' if options[:queues].empty?
297
-
298
- if !File.exist?(options[:require]) ||
299
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
323
+ if !File.exist?(@config[:require]) ||
324
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
300
325
  logger.info "=================================================================="
301
- logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
302
- logger.info " to load your worker classes with -r [DIR|FILE]."
326
+ logger.info " Please point Sidekiq to a Rails application or a Ruby file "
327
+ logger.info " to load your job classes with -r [DIR|FILE]."
303
328
  logger.info "=================================================================="
304
329
  logger.info @parser
305
330
  die(1)
306
331
  end
307
332
 
308
333
  [:concurrency, :timeout].each do |opt|
309
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
334
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
310
335
  end
311
336
  end
312
337
 
313
338
  def parse_options(argv)
314
339
  opts = {}
340
+ @parser = option_parser(opts)
341
+ @parser.parse!(argv)
342
+ opts
343
+ end
315
344
 
316
- @parser = OptionParser.new do |o|
317
- o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
345
+ def option_parser(opts)
346
+ parser = OptionParser.new { |o|
347
+ o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
318
348
  opts[:concurrency] = Integer(arg)
319
349
  end
320
350
 
321
- o.on '-d', '--daemon', "Daemonize process" do |arg|
322
- opts[:daemon] = arg
323
- end
324
-
325
- o.on '-e', '--environment ENV', "Application environment" do |arg|
351
+ o.on "-e", "--environment ENV", "Application environment" do |arg|
326
352
  opts[:environment] = arg
327
353
  end
328
354
 
329
- o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
355
+ o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
330
356
  opts[:tag] = arg
331
357
  end
332
358
 
333
- o.on '-i', '--index INT', "unique process index on this machine" do |arg|
334
- opts[:index] = Integer(arg.match(/\d+/)[0])
335
- end
336
-
337
359
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
338
- queue, weight = arg.split(",")
339
- parse_queue opts, queue, weight
360
+ opts[:queues] ||= []
361
+ opts[:queues] << arg
340
362
  end
341
363
 
342
- o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
364
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
343
365
  opts[:require] = arg
344
366
  end
345
367
 
346
- o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
368
+ o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
347
369
  opts[:timeout] = Integer(arg)
348
370
  end
349
371
 
@@ -351,88 +373,51 @@ module Sidekiq
351
373
  opts[:verbose] = arg
352
374
  end
353
375
 
354
- o.on '-C', '--config PATH', "path to YAML config file" do |arg|
376
+ o.on "-C", "--config PATH", "path to YAML config file" do |arg|
355
377
  opts[:config_file] = arg
356
378
  end
357
379
 
358
- o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
359
- opts[:logfile] = arg
360
- end
361
-
362
- o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
363
- opts[:pidfile] = arg
364
- end
365
-
366
- o.on '-V', '--version', "Print version and exit" do |arg|
380
+ o.on "-V", "--version", "Print version and exit" do
367
381
  puts "Sidekiq #{Sidekiq::VERSION}"
368
382
  die(0)
369
383
  end
370
- end
384
+ }
371
385
 
372
- @parser.banner = "sidekiq [options]"
373
- @parser.on_tail "-h", "--help", "Show help" do
374
- logger.info @parser
386
+ parser.banner = "sidekiq [options]"
387
+ parser.on_tail "-h", "--help", "Show help" do
388
+ logger.info parser
375
389
  die 1
376
390
  end
377
- @parser.parse!(argv)
378
391
 
379
- %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
380
- opts[:config_file] ||= filename if File.exist?(filename)
381
- end
382
-
383
- opts
392
+ parser
384
393
  end
385
394
 
386
395
  def initialize_logger
387
- Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
388
-
389
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
390
- end
391
-
392
- def write_pid
393
- if path = options[:pidfile]
394
- pidfile = File.expand_path(path)
395
- File.open(pidfile, 'w') do |f|
396
- f.puts ::Process.pid
397
- end
398
- end
396
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
399
397
  end
400
398
 
401
- def parse_config(cfile)
402
- opts = {}
403
- if File.exist?(cfile)
404
- opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
399
+ def parse_config(path)
400
+ erb = ERB.new(File.read(path), trim_mode: "-")
401
+ erb.filename = File.expand_path(path)
402
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
405
403
 
406
- if opts.respond_to? :deep_symbolize_keys!
407
- opts.deep_symbolize_keys!
408
- else
409
- symbolize_keys_deep!(opts)
410
- end
411
-
412
- opts = opts.merge(opts.delete(environment.to_sym) || {})
413
- parse_queues(opts, opts.delete(:queues) || [])
404
+ if opts.respond_to? :deep_symbolize_keys!
405
+ opts.deep_symbolize_keys!
414
406
  else
415
- # allow a non-existent config file so Sidekiq
416
- # can be deployed by cap with just the defaults.
417
- end
418
- ns = opts.delete(:namespace)
419
- if ns
420
- # logger hasn't been initialized yet, puts is all we have.
421
- puts("namespace should be set in your ruby initializer, is ignored in config file")
422
- puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
407
+ symbolize_keys_deep!(opts)
423
408
  end
424
- opts
425
- end
426
409
 
427
- def parse_queues(opts, queues_and_weights)
428
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
410
+ opts = opts.merge(opts.delete(environment.to_sym) || {})
411
+ opts.delete(:strict)
412
+
413
+ opts
429
414
  end
430
415
 
431
- def parse_queue(opts, q, weight=nil)
432
- [weight.to_i, 1].max.times do
433
- (opts[:queues] ||= []) << q
434
- end
435
- opts[:strict] = false if weight.to_i > 0
416
+ def rails_app?
417
+ defined?(::Rails) && ::Rails.respond_to?(:application)
436
418
  end
437
419
  end
438
420
  end
421
+
422
+ require "sidekiq/systemd"
423
+ require "sidekiq/metrics/tracking"