sidekiq 5.1.1 → 6.5.9

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (126) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +507 -1
  3. data/LICENSE +3 -3
  4. data/README.md +24 -35
  5. data/bin/sidekiq +27 -3
  6. data/bin/sidekiqload +80 -68
  7. data/bin/sidekiqmon +8 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +537 -286
  13. data/lib/sidekiq/cli.rb +243 -240
  14. data/lib/sidekiq/client.rb +82 -85
  15. data/lib/sidekiq/component.rb +65 -0
  16. data/lib/sidekiq/delay.rb +9 -7
  17. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  18. data/lib/sidekiq/extensions/active_record.rb +13 -10
  19. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  20. data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
  21. data/lib/sidekiq/fetch.rb +50 -40
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +36 -9
  24. data/lib/sidekiq/job_retry.rb +143 -97
  25. data/lib/sidekiq/job_util.rb +71 -0
  26. data/lib/sidekiq/launcher.rb +185 -85
  27. data/lib/sidekiq/logger.rb +156 -0
  28. data/lib/sidekiq/manager.rb +41 -43
  29. data/lib/sidekiq/metrics/deploy.rb +47 -0
  30. data/lib/sidekiq/metrics/query.rb +153 -0
  31. data/lib/sidekiq/metrics/shared.rb +94 -0
  32. data/lib/sidekiq/metrics/tracking.rb +134 -0
  33. data/lib/sidekiq/middleware/chain.rb +102 -46
  34. data/lib/sidekiq/middleware/current_attributes.rb +63 -0
  35. data/lib/sidekiq/middleware/i18n.rb +7 -7
  36. data/lib/sidekiq/middleware/modules.rb +21 -0
  37. data/lib/sidekiq/monitor.rb +133 -0
  38. data/lib/sidekiq/paginator.rb +28 -16
  39. data/lib/sidekiq/processor.rb +156 -98
  40. data/lib/sidekiq/rails.rb +48 -42
  41. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  42. data/lib/sidekiq/redis_connection.rb +109 -51
  43. data/lib/sidekiq/ring_buffer.rb +29 -0
  44. data/lib/sidekiq/scheduled.rb +133 -41
  45. data/lib/sidekiq/sd_notify.rb +149 -0
  46. data/lib/sidekiq/systemd.rb +24 -0
  47. data/lib/sidekiq/testing/inline.rb +6 -5
  48. data/lib/sidekiq/testing.rb +72 -62
  49. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  50. data/lib/sidekiq/version.rb +2 -1
  51. data/lib/sidekiq/web/action.rb +15 -11
  52. data/lib/sidekiq/web/application.rb +127 -76
  53. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  54. data/lib/sidekiq/web/helpers.rb +133 -96
  55. data/lib/sidekiq/web/router.rb +23 -19
  56. data/lib/sidekiq/web.rb +69 -109
  57. data/lib/sidekiq/worker.rb +268 -102
  58. data/lib/sidekiq.rb +175 -66
  59. data/sidekiq.gemspec +23 -23
  60. data/web/assets/images/apple-touch-icon.png +0 -0
  61. data/web/assets/javascripts/application.js +112 -61
  62. data/web/assets/javascripts/chart.min.js +13 -0
  63. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  64. data/web/assets/javascripts/dashboard.js +65 -91
  65. data/web/assets/javascripts/graph.js +16 -0
  66. data/web/assets/javascripts/metrics.js +262 -0
  67. data/web/assets/stylesheets/application-dark.css +143 -0
  68. data/web/assets/stylesheets/application-rtl.css +0 -4
  69. data/web/assets/stylesheets/application.css +120 -232
  70. data/web/assets/stylesheets/bootstrap.css +2 -2
  71. data/web/locales/ar.yml +9 -2
  72. data/web/locales/de.yml +14 -2
  73. data/web/locales/el.yml +43 -19
  74. data/web/locales/en.yml +14 -1
  75. data/web/locales/es.yml +21 -5
  76. data/web/locales/fr.yml +10 -3
  77. data/web/locales/ja.yml +14 -1
  78. data/web/locales/lt.yml +83 -0
  79. data/web/locales/pl.yml +4 -4
  80. data/web/locales/pt-br.yml +27 -9
  81. data/web/locales/ru.yml +4 -0
  82. data/web/locales/vi.yml +83 -0
  83. data/web/locales/zh-cn.yml +36 -11
  84. data/web/locales/zh-tw.yml +32 -7
  85. data/web/views/_footer.erb +4 -1
  86. data/web/views/_job_info.erb +3 -2
  87. data/web/views/_nav.erb +4 -18
  88. data/web/views/_poll_link.erb +2 -5
  89. data/web/views/_summary.erb +7 -7
  90. data/web/views/busy.erb +61 -22
  91. data/web/views/dashboard.erb +23 -14
  92. data/web/views/dead.erb +3 -3
  93. data/web/views/layout.erb +4 -2
  94. data/web/views/metrics.erb +69 -0
  95. data/web/views/metrics_for_job.erb +87 -0
  96. data/web/views/morgue.erb +9 -6
  97. data/web/views/queue.erb +24 -10
  98. data/web/views/queues.erb +11 -3
  99. data/web/views/retries.erb +14 -7
  100. data/web/views/retry.erb +3 -3
  101. data/web/views/scheduled.erb +5 -2
  102. metadata +62 -135
  103. data/.github/contributing.md +0 -32
  104. data/.github/issue_template.md +0 -11
  105. data/.gitignore +0 -13
  106. data/.travis.yml +0 -14
  107. data/3.0-Upgrade.md +0 -70
  108. data/4.0-Upgrade.md +0 -53
  109. data/5.0-Upgrade.md +0 -56
  110. data/COMM-LICENSE +0 -95
  111. data/Ent-Changes.md +0 -210
  112. data/Gemfile +0 -8
  113. data/Pro-2.0-Upgrade.md +0 -138
  114. data/Pro-3.0-Upgrade.md +0 -44
  115. data/Pro-4.0-Upgrade.md +0 -35
  116. data/Pro-Changes.md +0 -716
  117. data/Rakefile +0 -8
  118. data/bin/sidekiqctl +0 -99
  119. data/code_of_conduct.md +0 -50
  120. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  121. data/lib/sidekiq/core_ext.rb +0 -1
  122. data/lib/sidekiq/exception_handler.rb +0 -29
  123. data/lib/sidekiq/logging.rb +0 -122
  124. data/lib/sidekiq/middleware/server/active_record.rb +0 -22
  125. data/lib/sidekiq/middleware/server/active_record_cache.rb +0 -11
  126. data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/cli.rb CHANGED
@@ -1,46 +1,45 @@
1
- # encoding: utf-8
2
1
  # frozen_string_literal: true
3
- $stdout.sync = true
4
-
5
- require 'yaml'
6
- require 'singleton'
7
- require 'optparse'
8
- require 'erb'
9
- require 'fileutils'
10
2
 
11
- require 'sidekiq'
12
- require 'sidekiq/util'
3
+ $stdout.sync = true
13
4
 
14
- module Sidekiq
5
+ require "yaml"
6
+ require "singleton"
7
+ require "optparse"
8
+ require "erb"
9
+ require "fileutils"
10
+
11
+ require "sidekiq"
12
+ require "sidekiq/component"
13
+ require "sidekiq/launcher"
14
+
15
+ # module ScoutApm
16
+ # VERSION = "5.3.1"
17
+ # end
18
+ fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
+
20
+
21
+ scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
+
23
+
24
+ EOM
25
+
26
+ module Sidekiq # :nodoc:
15
27
  class CLI
16
- include Util
28
+ include Sidekiq::Component
17
29
  include Singleton unless $TESTING
18
30
 
19
- PROCTITLES = [
20
- proc { 'sidekiq'.freeze },
21
- proc { Sidekiq::VERSION },
22
- proc { |me, data| data['tag'] },
23
- proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
24
- proc { |me, data| "stopping" if me.stopping? },
25
- ]
26
-
27
- # Used for CLI testing
28
- attr_accessor :code
29
31
  attr_accessor :launcher
30
32
  attr_accessor :environment
33
+ attr_accessor :config
31
34
 
32
- def initialize
33
- @code = nil
34
- end
35
-
36
- def parse(args=ARGV)
37
- @code = nil
35
+ def parse(args = ARGV.dup)
36
+ @config = Sidekiq
37
+ @config[:error_handlers].clear
38
+ @config[:error_handlers] << @config.method(:default_error_handler)
38
39
 
39
40
  setup_options(args)
40
41
  initialize_logger
41
42
  validate!
42
- daemonize
43
- write_pid
44
43
  end
45
44
 
46
45
  def jruby?
@@ -50,183 +49,190 @@ module Sidekiq
50
49
  # Code within this method is not tested because it alters
51
50
  # global process state irreversibly. PRs which improve the
52
51
  # test coverage of Sidekiq::CLI are welcomed.
53
- def run
54
- boot_system
55
- print_banner
52
+ def run(boot_app: true)
53
+ boot_application if boot_app
56
54
 
57
- self_read, self_write = IO.pipe
58
- sigs = %w(INT TERM TTIN TSTP)
59
- # USR1 and USR2 don't work on the JVM
60
- if !jruby?
61
- sigs << 'USR1'
62
- sigs << 'USR2'
55
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
56
+ print_banner
63
57
  end
58
+ logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
64
59
 
60
+ self_read, self_write = IO.pipe
61
+ sigs = %w[INT TERM TTIN TSTP]
62
+ # USR1 and USR2 don't work on the JVM
63
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
65
64
  sigs.each do |sig|
66
- begin
67
- trap sig do
68
- self_write.puts(sig)
65
+ old_handler = Signal.trap(sig) do
66
+ if old_handler.respond_to?(:call)
67
+ begin
68
+ old_handler.call
69
+ rescue Exception => exc
70
+ # signal handlers can't use Logger so puts only
71
+ puts ["Error in #{sig} handler", exc].inspect
72
+ end
69
73
  end
70
- rescue ArgumentError
71
- puts "Signal #{sig} not supported"
74
+ self_write.puts(sig)
72
75
  end
76
+ rescue ArgumentError
77
+ puts "Signal #{sig} not supported"
73
78
  end
74
79
 
75
80
  logger.info "Running in #{RUBY_DESCRIPTION}"
76
81
  logger.info Sidekiq::LICENSE
77
- logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
82
+ logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
78
83
 
79
84
  # touch the connection pool so it is created before we
80
85
  # fire startup and start multithreading.
81
- ver = Sidekiq.redis_info['redis_version']
82
- raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
86
+ info = @config.redis_info
87
+ ver = info["redis_version"]
88
+ raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
89
+
90
+ maxmemory_policy = info["maxmemory_policy"]
91
+ if maxmemory_policy != "noeviction"
92
+ logger.warn <<~EOM
93
+
94
+
95
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
96
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
97
+ See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
98
+
99
+ EOM
100
+ end
101
+
102
+ # Since the user can pass us a connection pool explicitly in the initializer, we
103
+ # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
104
+ cursize = @config.redis_pool.size
105
+ needed = @config[:concurrency] + 2
106
+ raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
83
107
 
84
108
  # cache process identity
85
- Sidekiq.options[:identity] = identity
109
+ @config[:identity] = identity
86
110
 
87
111
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
88
- Sidekiq.server_middleware
112
+ @config.server_middleware
89
113
 
90
114
  # Before this point, the process is initializing with just the main thread.
91
115
  # Starting here the process will now have multiple threads running.
92
116
  fire_event(:startup, reverse: false, reraise: true)
93
117
 
94
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
95
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
118
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
119
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
120
+
121
+ launch(self_read)
122
+ end
96
123
 
97
- if !options[:daemon]
98
- logger.info 'Starting processing, hit Ctrl-C to stop'
124
+ def launch(self_read)
125
+ if environment == "development" && $stdout.tty?
126
+ logger.info "Starting processing, hit Ctrl-C to stop"
99
127
  end
100
128
 
101
- require 'sidekiq/launcher'
102
- @launcher = Sidekiq::Launcher.new(options)
129
+ @launcher = Sidekiq::Launcher.new(@config)
103
130
 
104
131
  begin
105
132
  launcher.run
106
133
 
107
- while readable_io = IO.select([self_read])
108
- signal = readable_io.first[0].gets.strip
134
+ while self_read.wait_readable
135
+ signal = self_read.gets.strip
109
136
  handle_signal(signal)
110
137
  end
111
138
  rescue Interrupt
112
- logger.info 'Shutting down'
139
+ logger.info "Shutting down"
113
140
  launcher.stop
114
- # Explicitly exit so busy Processor threads can't block
115
- # process shutdown.
116
141
  logger.info "Bye!"
142
+
143
+ # Explicitly exit so busy Processor threads won't block process shutdown.
144
+ #
145
+ # NB: slow at_exit handlers will prevent a timely exit if they take
146
+ # a while to run. If Sidekiq is getting here but the process isn't exiting,
147
+ # use the TTIN signal to determine where things are stuck.
117
148
  exit(0)
118
149
  end
119
150
  end
120
151
 
152
+ def self.w
153
+ "\e[37m"
154
+ end
155
+
156
+ def self.r
157
+ "\e[31m"
158
+ end
159
+
160
+ def self.b
161
+ "\e[30m"
162
+ end
163
+
164
+ def self.reset
165
+ "\e[0m"
166
+ end
167
+
121
168
  def self.banner
122
- %q{
123
- m,
124
- `$b
125
- .ss, $$: .,d$
126
- `$$P,d$P' .,md$P"'
127
- ,$$$$$bmmd$$$P^'
128
- .d$$$$$$$$$$P'
129
- $$^' `"^$$$' ____ _ _ _ _
130
- $: ,$$: / ___|(_) __| | ___| | _(_) __ _
131
- `b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
132
- $$: ___) | | (_| | __/ <| | (_| |
133
- $$ |____/|_|\__,_|\___|_|\_\_|\__, |
134
- .d$$ |_|
135
- }
169
+ %{
170
+ #{w} m,
171
+ #{w} `$b
172
+ #{w} .ss, $$: .,d$
173
+ #{w} `$$P,d$P' .,md$P"'
174
+ #{w} ,$$$$$b#{b}/#{w}md$$$P^'
175
+ #{w} .d$$$$$$#{b}/#{w}$$$P'
176
+ #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
177
+ #{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
178
+ #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
179
+ #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
180
+ #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
181
+ #{w} .d$$ #{r} |_|
182
+ #{reset}}
136
183
  end
137
184
 
138
185
  SIGNAL_HANDLERS = {
139
186
  # Ctrl-C in terminal
140
- 'INT' => ->(cli) { raise Interrupt },
187
+ "INT" => ->(cli) { raise Interrupt },
141
188
  # TERM is the signal that Sidekiq must exit.
142
189
  # Heroku sends TERM and then waits 30 seconds for process to exit.
143
- 'TERM' => ->(cli) { raise Interrupt },
144
- 'USR1' => ->(cli) {
145
- Sidekiq.logger.info "Received USR1, no longer accepting new work"
146
- cli.launcher.quiet
147
- },
148
- 'TSTP' => ->(cli) {
149
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
190
+ "TERM" => ->(cli) { raise Interrupt },
191
+ "TSTP" => ->(cli) {
192
+ cli.logger.info "Received TSTP, no longer accepting new work"
150
193
  cli.launcher.quiet
151
194
  },
152
- 'USR2' => ->(cli) {
153
- if Sidekiq.options[:logfile]
154
- Sidekiq.logger.info "Received USR2, reopening log file"
155
- Sidekiq::Logging.reopen_logs
156
- end
157
- },
158
- 'TTIN' => ->(cli) {
195
+ "TTIN" => ->(cli) {
159
196
  Thread.list.each do |thread|
160
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
197
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
161
198
  if thread.backtrace
162
- Sidekiq.logger.warn thread.backtrace.join("\n")
199
+ cli.logger.warn thread.backtrace.join("\n")
163
200
  else
164
- Sidekiq.logger.warn "<no backtrace available>"
201
+ cli.logger.warn "<no backtrace available>"
165
202
  end
166
203
  end
167
- },
204
+ }
168
205
  }
206
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
207
+ SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
169
208
 
170
209
  def handle_signal(sig)
171
- Sidekiq.logger.debug "Got #{sig} signal"
172
- handy = SIGNAL_HANDLERS[sig]
173
- if handy
174
- handy.call(self)
175
- else
176
- Sidekiq.logger.info { "No signal handler for #{sig}" }
177
- end
210
+ logger.debug "Got #{sig} signal"
211
+ SIGNAL_HANDLERS[sig].call(self)
178
212
  end
179
213
 
180
214
  private
181
215
 
182
216
  def print_banner
183
- # Print logo and banner for development
184
- if environment == 'development' && $stdout.tty?
185
- puts "\e[#{31}m"
186
- puts Sidekiq::CLI.banner
187
- puts "\e[0m"
188
- end
189
- end
190
-
191
- def daemonize
192
- return unless options[:daemon]
193
-
194
- raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
195
- files_to_reopen = []
196
- ObjectSpace.each_object(File) do |file|
197
- files_to_reopen << file unless file.closed?
198
- end
199
-
200
- ::Process.daemon(true, true)
201
-
202
- files_to_reopen.each do |file|
203
- begin
204
- file.reopen file.path, "a+"
205
- file.sync = true
206
- rescue ::Exception
207
- end
208
- end
209
-
210
- [$stdout, $stderr].each do |io|
211
- File.open(options[:logfile], 'ab') do |f|
212
- io.reopen(f)
213
- end
214
- io.sync = true
215
- end
216
- $stdin.reopen('/dev/null')
217
-
218
- initialize_logger
217
+ puts "\e[31m"
218
+ puts Sidekiq::CLI.banner
219
+ puts "\e[0m"
219
220
  end
220
221
 
221
222
  def set_environment(cli_env)
222
- @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
223
+ # See #984 for discussion.
224
+ # APP_ENV is now the preferred ENV term since it is not tech-specific.
225
+ # Both Sinatra 2.0+ and Sidekiq support this term.
226
+ # RAILS_ENV and RACK_ENV are there for legacy support.
227
+ @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
+ config[:environment] = @environment
223
229
  end
224
230
 
225
231
  def symbolize_keys_deep!(hash)
226
232
  hash.keys.each do |k|
227
233
  symkey = k.respond_to?(:to_sym) ? k.to_sym : k
228
234
  hash[symkey] = hash.delete k
229
- symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
235
+ symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
230
236
  end
231
237
  end
232
238
 
@@ -234,58 +240,63 @@ module Sidekiq
234
240
  alias_method :☠, :exit
235
241
 
236
242
  def setup_options(args)
243
+ # parse CLI options
237
244
  opts = parse_options(args)
245
+
238
246
  set_environment opts[:environment]
239
247
 
240
- cfile = opts[:config_file]
241
- opts = parse_config(cfile).merge(opts) if cfile
248
+ # check config file presence
249
+ if opts[:config_file]
250
+ unless File.exist?(opts[:config_file])
251
+ raise ArgumentError, "No such file #{opts[:config_file]}"
252
+ end
253
+ else
254
+ config_dir = if File.directory?(opts[:require].to_s)
255
+ File.join(opts[:require], "config")
256
+ else
257
+ File.join(@config[:require], "config")
258
+ end
242
259
 
243
- opts[:strict] = true if opts[:strict].nil?
244
- opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
260
+ %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
261
+ path = File.join(config_dir, config_file)
262
+ opts[:config_file] ||= path if File.exist?(path)
263
+ end
264
+ end
245
265
 
246
- options.merge!(opts)
247
- end
266
+ # parse config file options
267
+ opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
248
268
 
249
- def options
250
- Sidekiq.options
251
- end
269
+ # set defaults
270
+ opts[:queues] = ["default"] if opts[:queues].nil?
271
+ opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
252
272
 
253
- def boot_system
254
- ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
273
+ # merge with defaults
274
+ @config.merge!(opts)
275
+ end
255
276
 
256
- raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
277
+ def boot_application
278
+ ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
257
279
 
258
- if File.directory?(options[:require])
259
- require 'rails'
260
- if ::Rails::VERSION::MAJOR < 4
280
+ if File.directory?(@config[:require])
281
+ require "rails"
282
+ if ::Rails::VERSION::MAJOR < 5
261
283
  raise "Sidekiq no longer supports this version of Rails"
262
- elsif ::Rails::VERSION::MAJOR == 4
263
- # Painful contortions, see 1791 for discussion
264
- # No autoloading, we want to force eager load for everything.
265
- require File.expand_path("#{options[:require]}/config/application.rb")
266
- ::Rails::Application.initializer "sidekiq.eager_load" do
267
- ::Rails.application.config.eager_load = true
268
- end
269
- require 'sidekiq/rails'
270
- require File.expand_path("#{options[:require]}/config/environment.rb")
271
284
  else
272
- require 'sidekiq/rails'
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
285
+ require "sidekiq/rails"
286
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
274
287
  end
275
- options[:tag] ||= default_tag
288
+ @config[:tag] ||= default_tag
276
289
  else
277
- not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
278
- "./#{options[:require]} or /path/to/#{options[:require]}"
279
-
280
- require(options[:require]) || raise(ArgumentError, not_required_message)
290
+ require @config[:require]
281
291
  end
282
292
  end
283
293
 
284
294
  def default_tag
285
295
  dir = ::Rails.root
286
296
  name = File.basename(dir)
287
- if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
288
- if File.basename(prevdir) == 'releases'
297
+ prevdir = File.dirname(dir) # Capistrano release directory?
298
+ if name.to_i != 0 && prevdir
299
+ if File.basename(prevdir) == "releases"
289
300
  return File.basename(File.dirname(prevdir))
290
301
  end
291
302
  end
@@ -293,57 +304,56 @@ module Sidekiq
293
304
  end
294
305
 
295
306
  def validate!
296
- options[:queues] << 'default' if options[:queues].empty?
297
-
298
- if !File.exist?(options[:require]) ||
299
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
307
+ if !File.exist?(@config[:require]) ||
308
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
300
309
  logger.info "=================================================================="
301
- logger.info " Please point sidekiq to a Rails 4/5 application or a Ruby file "
302
- logger.info " to load your worker classes with -r [DIR|FILE]."
310
+ logger.info " Please point Sidekiq to a Rails application or a Ruby file "
311
+ logger.info " to load your job classes with -r [DIR|FILE]."
303
312
  logger.info "=================================================================="
304
313
  logger.info @parser
305
314
  die(1)
306
315
  end
307
316
 
308
317
  [:concurrency, :timeout].each do |opt|
309
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
318
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
310
319
  end
311
320
  end
312
321
 
313
322
  def parse_options(argv)
314
323
  opts = {}
324
+ @parser = option_parser(opts)
325
+ @parser.parse!(argv)
326
+ opts
327
+ end
315
328
 
316
- @parser = OptionParser.new do |o|
317
- o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
329
+ def option_parser(opts)
330
+ parser = OptionParser.new { |o|
331
+ o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
318
332
  opts[:concurrency] = Integer(arg)
319
333
  end
320
334
 
321
- o.on '-d', '--daemon', "Daemonize process" do |arg|
322
- opts[:daemon] = arg
335
+ o.on "-d", "--daemon", "Daemonize process" do |arg|
336
+ puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
323
337
  end
324
338
 
325
- o.on '-e', '--environment ENV', "Application environment" do |arg|
339
+ o.on "-e", "--environment ENV", "Application environment" do |arg|
326
340
  opts[:environment] = arg
327
341
  end
328
342
 
329
- o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
343
+ o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
330
344
  opts[:tag] = arg
331
345
  end
332
346
 
333
- o.on '-i', '--index INT', "unique process index on this machine" do |arg|
334
- opts[:index] = Integer(arg.match(/\d+/)[0])
335
- end
336
-
337
347
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
338
348
  queue, weight = arg.split(",")
339
349
  parse_queue opts, queue, weight
340
350
  end
341
351
 
342
- o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
352
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
343
353
  opts[:require] = arg
344
354
  end
345
355
 
346
- o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
356
+ o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
347
357
  opts[:timeout] = Integer(arg)
348
358
  end
349
359
 
@@ -351,88 +361,81 @@ module Sidekiq
351
361
  opts[:verbose] = arg
352
362
  end
353
363
 
354
- o.on '-C', '--config PATH', "path to YAML config file" do |arg|
364
+ o.on "-C", "--config PATH", "path to YAML config file" do |arg|
355
365
  opts[:config_file] = arg
356
366
  end
357
367
 
358
- o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
359
- opts[:logfile] = arg
368
+ o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
369
+ puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
360
370
  end
361
371
 
362
- o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
363
- opts[:pidfile] = arg
372
+ o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
373
+ puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
364
374
  end
365
375
 
366
- o.on '-V', '--version', "Print version and exit" do |arg|
376
+ o.on "-V", "--version", "Print version and exit" do |arg|
367
377
  puts "Sidekiq #{Sidekiq::VERSION}"
368
378
  die(0)
369
379
  end
370
- end
380
+ }
371
381
 
372
- @parser.banner = "sidekiq [options]"
373
- @parser.on_tail "-h", "--help", "Show help" do
374
- logger.info @parser
382
+ parser.banner = "sidekiq [options]"
383
+ parser.on_tail "-h", "--help", "Show help" do
384
+ logger.info parser
375
385
  die 1
376
386
  end
377
- @parser.parse!(argv)
378
-
379
- %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
380
- opts[:config_file] ||= filename if File.exist?(filename)
381
- end
382
387
 
383
- opts
388
+ parser
384
389
  end
385
390
 
386
391
  def initialize_logger
387
- Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
388
-
389
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
392
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
390
393
  end
391
394
 
392
- def write_pid
393
- if path = options[:pidfile]
394
- pidfile = File.expand_path(path)
395
- File.open(pidfile, 'w') do |f|
396
- f.puts ::Process.pid
397
- end
395
+ def parse_config(path)
396
+ erb = ERB.new(File.read(path))
397
+ erb.filename = File.expand_path(path)
398
+ opts = load_yaml(erb.result) || {}
399
+
400
+ if opts.respond_to? :deep_symbolize_keys!
401
+ opts.deep_symbolize_keys!
402
+ else
403
+ symbolize_keys_deep!(opts)
398
404
  end
399
- end
400
405
 
401
- def parse_config(cfile)
402
- opts = {}
403
- if File.exist?(cfile)
404
- opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
406
+ opts = opts.merge(opts.delete(environment.to_sym) || {})
407
+ opts.delete(:strict)
405
408
 
406
- if opts.respond_to? :deep_symbolize_keys!
407
- opts.deep_symbolize_keys!
408
- else
409
- symbolize_keys_deep!(opts)
410
- end
409
+ parse_queues(opts, opts.delete(:queues) || [])
410
+
411
+ opts
412
+ end
411
413
 
412
- opts = opts.merge(opts.delete(environment.to_sym) || {})
413
- parse_queues(opts, opts.delete(:queues) || [])
414
+ def load_yaml(src)
415
+ if Psych::VERSION > "4.0"
416
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
414
417
  else
415
- # allow a non-existent config file so Sidekiq
416
- # can be deployed by cap with just the defaults.
417
- end
418
- ns = opts.delete(:namespace)
419
- if ns
420
- # logger hasn't been initialized yet, puts is all we have.
421
- puts("namespace should be set in your ruby initializer, is ignored in config file")
422
- puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
418
+ YAML.load(src)
423
419
  end
424
- opts
425
420
  end
426
421
 
427
422
  def parse_queues(opts, queues_and_weights)
428
423
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
429
424
  end
430
425
 
431
- def parse_queue(opts, q, weight=nil)
432
- [weight.to_i, 1].max.times do
433
- (opts[:queues] ||= []) << q
434
- end
426
+ def parse_queue(opts, queue, weight = nil)
427
+ opts[:queues] ||= []
428
+ opts[:strict] = true if opts[:strict].nil?
429
+ raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
430
+ [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
435
431
  opts[:strict] = false if weight.to_i > 0
436
432
  end
433
+
434
+ def rails_app?
435
+ defined?(::Rails) && ::Rails.respond_to?(:application)
436
+ end
437
437
  end
438
438
  end
439
+
440
+ require "sidekiq/systemd"
441
+ require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]