sidekiq 5.1.3 → 7.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (157) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +756 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +48 -51
  5. data/bin/multi_queue_bench +271 -0
  6. data/bin/sidekiq +22 -3
  7. data/bin/sidekiqload +213 -115
  8. data/bin/sidekiqmon +11 -0
  9. data/lib/generators/sidekiq/job_generator.rb +59 -0
  10. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  11. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  12. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  13. data/lib/sidekiq/api.rb +640 -330
  14. data/lib/sidekiq/capsule.rb +132 -0
  15. data/lib/sidekiq/cli.rb +244 -257
  16. data/lib/sidekiq/client.rb +132 -103
  17. data/lib/sidekiq/component.rb +68 -0
  18. data/lib/sidekiq/config.rb +293 -0
  19. data/lib/sidekiq/deploy.rb +64 -0
  20. data/lib/sidekiq/embedded.rb +63 -0
  21. data/lib/sidekiq/fetch.rb +49 -42
  22. data/lib/sidekiq/iterable_job.rb +55 -0
  23. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  24. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  25. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  26. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  27. data/lib/sidekiq/job/iterable.rb +231 -0
  28. data/lib/sidekiq/job.rb +385 -0
  29. data/lib/sidekiq/job_logger.rb +49 -12
  30. data/lib/sidekiq/job_retry.rb +167 -103
  31. data/lib/sidekiq/job_util.rb +109 -0
  32. data/lib/sidekiq/launcher.rb +209 -102
  33. data/lib/sidekiq/logger.rb +131 -0
  34. data/lib/sidekiq/manager.rb +43 -46
  35. data/lib/sidekiq/metrics/query.rb +158 -0
  36. data/lib/sidekiq/metrics/shared.rb +97 -0
  37. data/lib/sidekiq/metrics/tracking.rb +148 -0
  38. data/lib/sidekiq/middleware/chain.rb +113 -56
  39. data/lib/sidekiq/middleware/current_attributes.rb +113 -0
  40. data/lib/sidekiq/middleware/i18n.rb +7 -7
  41. data/lib/sidekiq/middleware/modules.rb +23 -0
  42. data/lib/sidekiq/monitor.rb +147 -0
  43. data/lib/sidekiq/paginator.rb +28 -16
  44. data/lib/sidekiq/processor.rb +175 -112
  45. data/lib/sidekiq/rails.rb +54 -39
  46. data/lib/sidekiq/redis_client_adapter.rb +114 -0
  47. data/lib/sidekiq/redis_connection.rb +65 -86
  48. data/lib/sidekiq/ring_buffer.rb +31 -0
  49. data/lib/sidekiq/scheduled.rb +139 -48
  50. data/lib/sidekiq/sd_notify.rb +149 -0
  51. data/lib/sidekiq/systemd.rb +26 -0
  52. data/lib/sidekiq/testing/inline.rb +6 -5
  53. data/lib/sidekiq/testing.rb +95 -94
  54. data/lib/sidekiq/transaction_aware_client.rb +51 -0
  55. data/lib/sidekiq/version.rb +3 -1
  56. data/lib/sidekiq/web/action.rb +22 -12
  57. data/lib/sidekiq/web/application.rb +225 -76
  58. data/lib/sidekiq/web/csrf_protection.rb +183 -0
  59. data/lib/sidekiq/web/helpers.rb +215 -118
  60. data/lib/sidekiq/web/router.rb +23 -19
  61. data/lib/sidekiq/web.rb +114 -106
  62. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  63. data/lib/sidekiq.rb +95 -182
  64. data/sidekiq.gemspec +26 -23
  65. data/web/assets/images/apple-touch-icon.png +0 -0
  66. data/web/assets/javascripts/application.js +157 -61
  67. data/web/assets/javascripts/base-charts.js +106 -0
  68. data/web/assets/javascripts/chart.min.js +13 -0
  69. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  70. data/web/assets/javascripts/dashboard-charts.js +192 -0
  71. data/web/assets/javascripts/dashboard.js +35 -283
  72. data/web/assets/javascripts/metrics.js +298 -0
  73. data/web/assets/stylesheets/application-dark.css +147 -0
  74. data/web/assets/stylesheets/application-rtl.css +10 -93
  75. data/web/assets/stylesheets/application.css +169 -522
  76. data/web/assets/stylesheets/bootstrap.css +2 -2
  77. data/web/locales/ar.yml +71 -64
  78. data/web/locales/cs.yml +62 -62
  79. data/web/locales/da.yml +60 -53
  80. data/web/locales/de.yml +65 -53
  81. data/web/locales/el.yml +43 -24
  82. data/web/locales/en.yml +86 -65
  83. data/web/locales/es.yml +70 -54
  84. data/web/locales/fa.yml +65 -65
  85. data/web/locales/fr.yml +83 -62
  86. data/web/locales/gd.yml +99 -0
  87. data/web/locales/he.yml +65 -64
  88. data/web/locales/hi.yml +59 -59
  89. data/web/locales/it.yml +53 -53
  90. data/web/locales/ja.yml +75 -64
  91. data/web/locales/ko.yml +52 -52
  92. data/web/locales/lt.yml +83 -0
  93. data/web/locales/nb.yml +61 -61
  94. data/web/locales/nl.yml +52 -52
  95. data/web/locales/pl.yml +45 -45
  96. data/web/locales/pt-br.yml +83 -55
  97. data/web/locales/pt.yml +51 -51
  98. data/web/locales/ru.yml +68 -63
  99. data/web/locales/sv.yml +53 -53
  100. data/web/locales/ta.yml +60 -60
  101. data/web/locales/tr.yml +101 -0
  102. data/web/locales/uk.yml +62 -61
  103. data/web/locales/ur.yml +64 -64
  104. data/web/locales/vi.yml +83 -0
  105. data/web/locales/zh-cn.yml +43 -16
  106. data/web/locales/zh-tw.yml +42 -8
  107. data/web/views/_footer.erb +18 -3
  108. data/web/views/_job_info.erb +21 -4
  109. data/web/views/_metrics_period_select.erb +12 -0
  110. data/web/views/_nav.erb +4 -18
  111. data/web/views/_paging.erb +2 -0
  112. data/web/views/_poll_link.erb +3 -6
  113. data/web/views/_summary.erb +7 -7
  114. data/web/views/busy.erb +79 -29
  115. data/web/views/dashboard.erb +49 -19
  116. data/web/views/dead.erb +3 -3
  117. data/web/views/filtering.erb +7 -0
  118. data/web/views/layout.erb +9 -7
  119. data/web/views/metrics.erb +91 -0
  120. data/web/views/metrics_for_job.erb +59 -0
  121. data/web/views/morgue.erb +14 -15
  122. data/web/views/queue.erb +33 -23
  123. data/web/views/queues.erb +19 -5
  124. data/web/views/retries.erb +19 -16
  125. data/web/views/retry.erb +3 -3
  126. data/web/views/scheduled.erb +17 -15
  127. metadata +84 -129
  128. data/.github/contributing.md +0 -32
  129. data/.github/issue_template.md +0 -11
  130. data/.gitignore +0 -13
  131. data/.travis.yml +0 -14
  132. data/3.0-Upgrade.md +0 -70
  133. data/4.0-Upgrade.md +0 -53
  134. data/5.0-Upgrade.md +0 -56
  135. data/COMM-LICENSE +0 -95
  136. data/Ent-Changes.md +0 -216
  137. data/Gemfile +0 -8
  138. data/LICENSE +0 -9
  139. data/Pro-2.0-Upgrade.md +0 -138
  140. data/Pro-3.0-Upgrade.md +0 -44
  141. data/Pro-4.0-Upgrade.md +0 -35
  142. data/Pro-Changes.md +0 -729
  143. data/Rakefile +0 -8
  144. data/bin/sidekiqctl +0 -99
  145. data/code_of_conduct.md +0 -50
  146. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  147. data/lib/sidekiq/core_ext.rb +0 -1
  148. data/lib/sidekiq/delay.rb +0 -42
  149. data/lib/sidekiq/exception_handler.rb +0 -29
  150. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  151. data/lib/sidekiq/extensions/active_record.rb +0 -40
  152. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  153. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  154. data/lib/sidekiq/logging.rb +0 -122
  155. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  156. data/lib/sidekiq/util.rb +0 -66
  157. data/lib/sidekiq/worker.rb +0 -204
data/lib/sidekiq/cli.rb CHANGED
@@ -1,45 +1,34 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  $stdout.sync = true
3
4
 
4
- require 'yaml'
5
- require 'singleton'
6
- require 'optparse'
7
- require 'erb'
8
- require 'fileutils'
5
+ require "yaml"
6
+ require "singleton"
7
+ require "optparse"
8
+ require "erb"
9
+ require "fileutils"
9
10
 
10
- require 'sidekiq'
11
- require 'sidekiq/util'
11
+ require "sidekiq"
12
+ require "sidekiq/config"
13
+ require "sidekiq/component"
14
+ require "sidekiq/capsule"
15
+ require "sidekiq/launcher"
12
16
 
13
- module Sidekiq
17
+ module Sidekiq # :nodoc:
14
18
  class CLI
15
- include Util
19
+ include Sidekiq::Component
16
20
  include Singleton unless $TESTING
17
21
 
18
- PROCTITLES = [
19
- proc { 'sidekiq' },
20
- proc { Sidekiq::VERSION },
21
- proc { |me, data| data['tag'] },
22
- proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
23
- proc { |me, data| "stopping" if me.stopping? },
24
- ]
25
-
26
- # Used for CLI testing
27
- attr_accessor :code
28
22
  attr_accessor :launcher
29
23
  attr_accessor :environment
24
+ attr_accessor :config
30
25
 
31
- def initialize
32
- @code = nil
33
- end
34
-
35
- def parse(args=ARGV)
36
- @code = nil
26
+ def parse(args = ARGV.dup)
27
+ @config ||= Sidekiq.default_configuration
37
28
 
38
29
  setup_options(args)
39
30
  initialize_logger
40
31
  validate!
41
- daemonize
42
- write_pid
43
32
  end
44
33
 
45
34
  def jruby?
@@ -49,183 +38,208 @@ module Sidekiq
49
38
  # Code within this method is not tested because it alters
50
39
  # global process state irreversibly. PRs which improve the
51
40
  # test coverage of Sidekiq::CLI are welcomed.
52
- def run
53
- boot_system
54
- print_banner
41
+ def run(boot_app: true, warmup: true)
42
+ boot_application if boot_app
55
43
 
56
- self_read, self_write = IO.pipe
57
- sigs = %w(INT TERM TTIN TSTP)
58
- # USR1 and USR2 don't work on the JVM
59
- if !jruby?
60
- sigs << 'USR1'
61
- sigs << 'USR2'
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
45
+ print_banner
62
46
  end
47
+ logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
63
48
 
49
+ self_read, self_write = IO.pipe
50
+ sigs = %w[INT TERM TTIN TSTP]
51
+ # USR1 and USR2 don't work on the JVM
52
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
64
53
  sigs.each do |sig|
65
- begin
66
- trap sig do
67
- self_write.write("#{sig}\n")
54
+ old_handler = Signal.trap(sig) do
55
+ if old_handler.respond_to?(:call)
56
+ begin
57
+ old_handler.call
58
+ rescue Exception => exc
59
+ # signal handlers can't use Logger so puts only
60
+ puts ["Error in #{sig} handler", exc].inspect
61
+ end
68
62
  end
69
- rescue ArgumentError
70
- puts "Signal #{sig} not supported"
63
+ self_write.puts(sig)
71
64
  end
65
+ rescue ArgumentError
66
+ puts "Signal #{sig} not supported"
72
67
  end
73
68
 
74
69
  logger.info "Running in #{RUBY_DESCRIPTION}"
75
70
  logger.info Sidekiq::LICENSE
76
- logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
71
+ logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
77
72
 
78
73
  # touch the connection pool so it is created before we
79
74
  # fire startup and start multithreading.
80
- ver = Sidekiq.redis_info['redis_version']
81
- raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
75
+ info = @config.redis_info
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
78
+
79
+ maxmemory_policy = info["maxmemory_policy"]
80
+ if maxmemory_policy != "noeviction" && maxmemory_policy != ""
81
+ # Redis Enterprise Cloud returns "" for their policy 😳
82
+ logger.warn <<~EOM
83
+
84
+
85
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
86
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
87
+ See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
88
+
89
+ EOM
90
+ end
91
+
92
+ # Since the user can pass us a connection pool explicitly in the initializer, we
93
+ # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
94
+ @config.capsules.each_pair do |name, cap|
95
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
96
+ end
82
97
 
83
98
  # cache process identity
84
- Sidekiq.options[:identity] = identity
99
+ @config[:identity] = identity
85
100
 
86
101
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
87
- Sidekiq.server_middleware
102
+ @config.server_middleware
103
+
104
+ ::Process.warmup if warmup && ::Process.respond_to?(:warmup)
88
105
 
89
106
  # Before this point, the process is initializing with just the main thread.
90
107
  # Starting here the process will now have multiple threads running.
91
108
  fire_event(:startup, reverse: false, reraise: true)
92
109
 
93
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
94
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
110
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
111
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
112
+
113
+ launch(self_read)
114
+ end
95
115
 
96
- if !options[:daemon]
97
- logger.info 'Starting processing, hit Ctrl-C to stop'
116
+ def launch(self_read)
117
+ if environment == "development" && $stdout.tty?
118
+ logger.info "Starting processing, hit Ctrl-C to stop"
98
119
  end
99
120
 
100
- require 'sidekiq/launcher'
101
- @launcher = Sidekiq::Launcher.new(options)
121
+ @launcher = Sidekiq::Launcher.new(@config)
102
122
 
103
123
  begin
104
124
  launcher.run
105
125
 
106
- while readable_io = IO.select([self_read])
107
- signal = readable_io.first[0].gets.strip
126
+ while self_read.wait_readable
127
+ signal = self_read.gets.strip
108
128
  handle_signal(signal)
109
129
  end
110
130
  rescue Interrupt
111
- logger.info 'Shutting down'
131
+ logger.info "Shutting down"
112
132
  launcher.stop
113
- # Explicitly exit so busy Processor threads can't block
114
- # process shutdown.
115
133
  logger.info "Bye!"
134
+
135
+ # Explicitly exit so busy Processor threads won't block process shutdown.
136
+ #
137
+ # NB: slow at_exit handlers will prevent a timely exit if they take
138
+ # a while to run. If Sidekiq is getting here but the process isn't exiting,
139
+ # use the TTIN signal to determine where things are stuck.
116
140
  exit(0)
117
141
  end
118
142
  end
119
143
 
144
+ HOLIDAY_COLORS = {
145
+ # got other color-specific holidays from around the world?
146
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
147
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
148
+ "10-31" => "\e[38;5;208m" # Halloween orange
149
+ }
150
+
151
+ def self.day
152
+ @@day ||= begin
153
+ t = Date.today
154
+ "#{t.month}-#{t.day}"
155
+ end
156
+ end
157
+
158
+ def self.r
159
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
160
+ end
161
+
162
+ def self.b
163
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
164
+ end
165
+
166
+ def self.w
167
+ "\e[1;37m"
168
+ end
169
+
170
+ def self.reset
171
+ @@b = @@r = @@day = nil
172
+ "\e[0m"
173
+ end
174
+
120
175
  def self.banner
121
- %q{
122
- m,
123
- `$b
124
- .ss, $$: .,d$
125
- `$$P,d$P' .,md$P"'
126
- ,$$$$$bmmd$$$P^'
127
- .d$$$$$$$$$$P'
128
- $$^' `"^$$$' ____ _ _ _ _
129
- $: ,$$: / ___|(_) __| | ___| | _(_) __ _
130
- `b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
131
- $$: ___) | | (_| | __/ <| | (_| |
132
- $$ |____/|_|\__,_|\___|_|\_\_|\__, |
133
- .d$$ |_|
134
- }
176
+ %{
177
+ #{w} m,
178
+ #{w} `$b
179
+ #{w} .ss, $$: .,d$
180
+ #{w} `$$P,d$P' .,md$P"'
181
+ #{w} ,$$$$$b#{b}/#{w}md$$$P^'
182
+ #{w} .d$$$$$$#{b}/#{w}$$$P'
183
+ #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
184
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
185
+ #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
186
+ #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
187
+ #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
188
+ #{w} .d$$ #{r} |_|
189
+ #{reset}}
135
190
  end
136
191
 
137
192
  SIGNAL_HANDLERS = {
138
193
  # Ctrl-C in terminal
139
- 'INT' => ->(cli) { raise Interrupt },
194
+ "INT" => ->(cli) { raise Interrupt },
140
195
  # TERM is the signal that Sidekiq must exit.
141
196
  # Heroku sends TERM and then waits 30 seconds for process to exit.
142
- 'TERM' => ->(cli) { raise Interrupt },
143
- 'USR1' => ->(cli) {
144
- Sidekiq.logger.info "Received USR1, no longer accepting new work"
145
- cli.launcher.quiet
146
- },
147
- 'TSTP' => ->(cli) {
148
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
197
+ "TERM" => ->(cli) { raise Interrupt },
198
+ "TSTP" => ->(cli) {
199
+ cli.logger.info "Received TSTP, no longer accepting new work"
149
200
  cli.launcher.quiet
150
201
  },
151
- 'USR2' => ->(cli) {
152
- if Sidekiq.options[:logfile]
153
- Sidekiq.logger.info "Received USR2, reopening log file"
154
- Sidekiq::Logging.reopen_logs
155
- end
156
- },
157
- 'TTIN' => ->(cli) {
202
+ "TTIN" => ->(cli) {
158
203
  Thread.list.each do |thread|
159
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
204
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
160
205
  if thread.backtrace
161
- Sidekiq.logger.warn thread.backtrace.join("\n")
206
+ cli.logger.warn thread.backtrace.join("\n")
162
207
  else
163
- Sidekiq.logger.warn "<no backtrace available>"
208
+ cli.logger.warn "<no backtrace available>"
164
209
  end
165
210
  end
166
- },
211
+ }
167
212
  }
213
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
214
+ SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
168
215
 
169
216
  def handle_signal(sig)
170
- Sidekiq.logger.debug "Got #{sig} signal"
171
- handy = SIGNAL_HANDLERS[sig]
172
- if handy
173
- handy.call(self)
174
- else
175
- Sidekiq.logger.info { "No signal handler for #{sig}" }
176
- end
217
+ logger.debug "Got #{sig} signal"
218
+ SIGNAL_HANDLERS[sig].call(self)
177
219
  end
178
220
 
179
221
  private
180
222
 
181
223
  def print_banner
182
- # Print logo and banner for development
183
- if environment == 'development' && $stdout.tty?
184
- puts "\e[#{31}m"
185
- puts Sidekiq::CLI.banner
186
- puts "\e[0m"
187
- end
188
- end
189
-
190
- def daemonize
191
- return unless options[:daemon]
192
-
193
- raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
194
- files_to_reopen = []
195
- ObjectSpace.each_object(File) do |file|
196
- files_to_reopen << file unless file.closed?
197
- end
198
-
199
- ::Process.daemon(true, true)
200
-
201
- files_to_reopen.each do |file|
202
- begin
203
- file.reopen file.path, "a+"
204
- file.sync = true
205
- rescue ::Exception
206
- end
207
- end
208
-
209
- [$stdout, $stderr].each do |io|
210
- File.open(options[:logfile], 'ab') do |f|
211
- io.reopen(f)
212
- end
213
- io.sync = true
214
- end
215
- $stdin.reopen('/dev/null')
216
-
217
- initialize_logger
224
+ puts "\e[31m"
225
+ puts Sidekiq::CLI.banner
226
+ puts "\e[0m"
218
227
  end
219
228
 
220
229
  def set_environment(cli_env)
221
- @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
230
+ # See #984 for discussion.
231
+ # APP_ENV is now the preferred ENV term since it is not tech-specific.
232
+ # Both Sinatra 2.0+ and Sidekiq support this term.
233
+ # RAILS_ENV and RACK_ENV are there for legacy support.
234
+ @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
235
+ config[:environment] = @environment
222
236
  end
223
237
 
224
238
  def symbolize_keys_deep!(hash)
225
239
  hash.keys.each do |k|
226
240
  symkey = k.respond_to?(:to_sym) ? k.to_sym : k
227
241
  hash[symkey] = hash.delete k
228
- symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
242
+ symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
229
243
  end
230
244
  end
231
245
 
@@ -233,58 +247,74 @@ module Sidekiq
233
247
  alias_method :☠, :exit
234
248
 
235
249
  def setup_options(args)
250
+ # parse CLI options
236
251
  opts = parse_options(args)
252
+
237
253
  set_environment opts[:environment]
238
254
 
239
- cfile = opts[:config_file]
240
- opts = parse_config(cfile).merge(opts) if cfile
255
+ # check config file presence
256
+ if opts[:config_file]
257
+ unless File.exist?(opts[:config_file])
258
+ raise ArgumentError, "No such file #{opts[:config_file]}"
259
+ end
260
+ else
261
+ config_dir = if File.directory?(opts[:require].to_s)
262
+ File.join(opts[:require], "config")
263
+ else
264
+ File.join(@config[:require], "config")
265
+ end
241
266
 
242
- opts[:strict] = true if opts[:strict].nil?
243
- opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
267
+ %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
268
+ path = File.join(config_dir, config_file)
269
+ opts[:config_file] ||= path if File.exist?(path)
270
+ end
271
+ end
244
272
 
245
- options.merge!(opts)
246
- end
273
+ # parse config file options
274
+ opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
275
+
276
+ # set defaults
277
+ opts[:queues] = ["default"] if opts[:queues].nil?
278
+ opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
279
+
280
+ # merge with defaults
281
+ @config.merge!(opts)
247
282
 
248
- def options
249
- Sidekiq.options
283
+ @config.default_capsule.tap do |cap|
284
+ cap.queues = opts[:queues]
285
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
286
+ end
287
+
288
+ opts[:capsules]&.each do |name, cap_config|
289
+ @config.capsule(name.to_s) do |cap|
290
+ cap.queues = cap_config[:queues]
291
+ cap.concurrency = cap_config[:concurrency]
292
+ end
293
+ end
250
294
  end
251
295
 
252
- def boot_system
253
- ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
254
-
255
- raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
256
-
257
- if File.directory?(options[:require])
258
- require 'rails'
259
- if ::Rails::VERSION::MAJOR < 4
260
- raise "Sidekiq no longer supports this version of Rails"
261
- elsif ::Rails::VERSION::MAJOR == 4
262
- # Painful contortions, see 1791 for discussion
263
- # No autoloading, we want to force eager load for everything.
264
- require File.expand_path("#{options[:require]}/config/application.rb")
265
- ::Rails::Application.initializer "sidekiq.eager_load" do
266
- ::Rails.application.config.eager_load = true
267
- end
268
- require 'sidekiq/rails'
269
- require File.expand_path("#{options[:require]}/config/environment.rb")
270
- else
271
- require 'sidekiq/rails'
272
- require File.expand_path("#{options[:require]}/config/environment.rb")
296
+ def boot_application
297
+ ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
298
+
299
+ if File.directory?(@config[:require])
300
+ require "rails"
301
+ if ::Rails::VERSION::MAJOR < 6
302
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
273
303
  end
274
- options[:tag] ||= default_tag
304
+ require "sidekiq/rails"
305
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
306
+ @config[:tag] ||= default_tag
275
307
  else
276
- not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
277
- "./#{options[:require]} or /path/to/#{options[:require]}"
278
-
279
- require(options[:require]) || raise(ArgumentError, not_required_message)
308
+ require @config[:require]
280
309
  end
281
310
  end
282
311
 
283
312
  def default_tag
284
313
  dir = ::Rails.root
285
314
  name = File.basename(dir)
286
- if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
287
- if File.basename(prevdir) == 'releases'
315
+ prevdir = File.dirname(dir) # Capistrano release directory?
316
+ if name.to_i != 0 && prevdir
317
+ if File.basename(prevdir) == "releases"
288
318
  return File.basename(File.dirname(prevdir))
289
319
  end
290
320
  end
@@ -292,59 +322,52 @@ module Sidekiq
292
322
  end
293
323
 
294
324
  def validate!
295
- options[:queues] << 'default' if options[:queues].empty?
296
-
297
- if !File.exist?(options[:require]) ||
298
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
325
+ if !File.exist?(@config[:require]) ||
326
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
299
327
  logger.info "=================================================================="
300
- logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
301
- logger.info " to load your worker classes with -r [DIR|FILE]."
328
+ logger.info " Please point Sidekiq to a Rails application or a Ruby file "
329
+ logger.info " to load your job classes with -r [DIR|FILE]."
302
330
  logger.info "=================================================================="
303
331
  logger.info @parser
304
332
  die(1)
305
333
  end
306
334
 
307
335
  [:concurrency, :timeout].each do |opt|
308
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
336
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
309
337
  end
310
338
  end
311
339
 
312
340
  def parse_options(argv)
313
341
  opts = {}
342
+ @parser = option_parser(opts)
343
+ @parser.parse!(argv)
344
+ opts
345
+ end
314
346
 
315
- @parser = OptionParser.new do |o|
316
- o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
347
+ def option_parser(opts)
348
+ parser = OptionParser.new { |o|
349
+ o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
317
350
  opts[:concurrency] = Integer(arg)
318
351
  end
319
352
 
320
- o.on '-d', '--daemon', "Daemonize process" do |arg|
321
- opts[:daemon] = arg
322
- end
323
-
324
- o.on '-e', '--environment ENV', "Application environment" do |arg|
353
+ o.on "-e", "--environment ENV", "Application environment" do |arg|
325
354
  opts[:environment] = arg
326
355
  end
327
356
 
328
- o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
357
+ o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
329
358
  opts[:tag] = arg
330
359
  end
331
360
 
332
- # this index remains here for backwards compatibility but none of the Sidekiq
333
- # family use this value anymore. it was used by Pro's original reliable_fetch.
334
- o.on '-i', '--index INT', "unique process index on this machine" do |arg|
335
- opts[:index] = Integer(arg.match(/\d+/)[0])
336
- end
337
-
338
361
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
339
- queue, weight = arg.split(",")
340
- parse_queue opts, queue, weight
362
+ opts[:queues] ||= []
363
+ opts[:queues] << arg
341
364
  end
342
365
 
343
- o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
366
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
344
367
  opts[:require] = arg
345
368
  end
346
369
 
347
- o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
370
+ o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
348
371
  opts[:timeout] = Integer(arg)
349
372
  end
350
373
 
@@ -352,88 +375,52 @@ module Sidekiq
352
375
  opts[:verbose] = arg
353
376
  end
354
377
 
355
- o.on '-C', '--config PATH', "path to YAML config file" do |arg|
378
+ o.on "-C", "--config PATH", "path to YAML config file" do |arg|
356
379
  opts[:config_file] = arg
357
380
  end
358
381
 
359
- o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
360
- opts[:logfile] = arg
361
- end
362
-
363
- o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
364
- opts[:pidfile] = arg
365
- end
366
-
367
- o.on '-V', '--version', "Print version and exit" do |arg|
382
+ o.on "-V", "--version", "Print version and exit" do
368
383
  puts "Sidekiq #{Sidekiq::VERSION}"
369
384
  die(0)
370
385
  end
371
- end
386
+ }
372
387
 
373
- @parser.banner = "sidekiq [options]"
374
- @parser.on_tail "-h", "--help", "Show help" do
375
- logger.info @parser
388
+ parser.banner = "sidekiq [options]"
389
+ parser.on_tail "-h", "--help", "Show help" do
390
+ logger.info parser
376
391
  die 1
377
392
  end
378
- @parser.parse!(argv)
379
393
 
380
- %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
381
- opts[:config_file] ||= filename if File.exist?(filename)
382
- end
383
-
384
- opts
394
+ parser
385
395
  end
386
396
 
387
397
  def initialize_logger
388
- Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
389
-
390
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
391
- end
392
-
393
- def write_pid
394
- if path = options[:pidfile]
395
- pidfile = File.expand_path(path)
396
- File.open(pidfile, 'w') do |f|
397
- f.puts ::Process.pid
398
- end
399
- end
398
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
400
399
  end
401
400
 
402
- def parse_config(cfile)
403
- opts = {}
404
- if File.exist?(cfile)
405
- opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
401
+ def parse_config(path)
402
+ erb = ERB.new(File.read(path), trim_mode: "-")
403
+ erb.filename = File.expand_path(path)
404
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
406
405
 
407
- if opts.respond_to? :deep_symbolize_keys!
408
- opts.deep_symbolize_keys!
409
- else
410
- symbolize_keys_deep!(opts)
411
- end
412
-
413
- opts = opts.merge(opts.delete(environment.to_sym) || {})
414
- parse_queues(opts, opts.delete(:queues) || [])
406
+ if opts.respond_to? :deep_symbolize_keys!
407
+ opts.deep_symbolize_keys!
415
408
  else
416
- # allow a non-existent config file so Sidekiq
417
- # can be deployed by cap with just the defaults.
418
- end
419
- ns = opts.delete(:namespace)
420
- if ns
421
- # logger hasn't been initialized yet, puts is all we have.
422
- puts("namespace should be set in your ruby initializer, is ignored in config file")
423
- puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
409
+ symbolize_keys_deep!(opts)
424
410
  end
425
- opts
426
- end
427
411
 
428
- def parse_queues(opts, queues_and_weights)
429
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
412
+ opts = opts.merge(opts.delete(environment.to_sym) || {})
413
+ opts.delete(:strict)
414
+
415
+ opts
430
416
  end
431
417
 
432
- def parse_queue(opts, q, weight=nil)
433
- [weight.to_i, 1].max.times do
434
- (opts[:queues] ||= []) << q
435
- end
436
- opts[:strict] = false if weight.to_i > 0
418
+ def rails_app?
419
+ defined?(::Rails) && ::Rails.respond_to?(:application)
437
420
  end
438
421
  end
439
422
  end
423
+
424
+ require "sidekiq/systemd"
425
+ require "sidekiq/metrics/tracking"
426
+ require "sidekiq/job/interrupt_handler"