sidekiq 6.5.1 → 7.3.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +376 -12
  3. data/README.md +43 -35
  4. data/bin/multi_queue_bench +271 -0
  5. data/bin/sidekiq +3 -8
  6. data/bin/sidekiqload +213 -118
  7. data/bin/sidekiqmon +3 -0
  8. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +88 -0
  9. data/lib/generators/sidekiq/job_generator.rb +2 -0
  10. data/lib/sidekiq/api.rb +378 -173
  11. data/lib/sidekiq/capsule.rb +132 -0
  12. data/lib/sidekiq/cli.rb +61 -63
  13. data/lib/sidekiq/client.rb +89 -40
  14. data/lib/sidekiq/component.rb +6 -2
  15. data/lib/sidekiq/config.rb +305 -0
  16. data/lib/sidekiq/deploy.rb +64 -0
  17. data/lib/sidekiq/embedded.rb +63 -0
  18. data/lib/sidekiq/fetch.rb +11 -14
  19. data/lib/sidekiq/iterable_job.rb +55 -0
  20. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  21. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  22. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  23. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  24. data/lib/sidekiq/job/iterable.rb +294 -0
  25. data/lib/sidekiq/job.rb +382 -10
  26. data/lib/sidekiq/job_logger.rb +8 -7
  27. data/lib/sidekiq/job_retry.rb +89 -46
  28. data/lib/sidekiq/job_util.rb +53 -15
  29. data/lib/sidekiq/launcher.rb +77 -69
  30. data/lib/sidekiq/logger.rb +2 -27
  31. data/lib/sidekiq/manager.rb +9 -11
  32. data/lib/sidekiq/metrics/query.rb +158 -0
  33. data/lib/sidekiq/metrics/shared.rb +106 -0
  34. data/lib/sidekiq/metrics/tracking.rb +148 -0
  35. data/lib/sidekiq/middleware/chain.rb +84 -48
  36. data/lib/sidekiq/middleware/current_attributes.rb +87 -20
  37. data/lib/sidekiq/middleware/modules.rb +2 -0
  38. data/lib/sidekiq/monitor.rb +19 -5
  39. data/lib/sidekiq/paginator.rb +11 -3
  40. data/lib/sidekiq/processor.rb +67 -56
  41. data/lib/sidekiq/rails.rb +22 -16
  42. data/lib/sidekiq/redis_client_adapter.rb +31 -71
  43. data/lib/sidekiq/redis_connection.rb +44 -117
  44. data/lib/sidekiq/ring_buffer.rb +2 -0
  45. data/lib/sidekiq/scheduled.rb +62 -35
  46. data/lib/sidekiq/systemd.rb +2 -0
  47. data/lib/sidekiq/testing.rb +37 -46
  48. data/lib/sidekiq/transaction_aware_client.rb +11 -5
  49. data/lib/sidekiq/version.rb +6 -1
  50. data/lib/sidekiq/web/action.rb +15 -5
  51. data/lib/sidekiq/web/application.rb +94 -24
  52. data/lib/sidekiq/web/csrf_protection.rb +10 -7
  53. data/lib/sidekiq/web/helpers.rb +118 -45
  54. data/lib/sidekiq/web/router.rb +5 -2
  55. data/lib/sidekiq/web.rb +67 -15
  56. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  57. data/lib/sidekiq.rb +78 -266
  58. data/sidekiq.gemspec +12 -10
  59. data/web/assets/javascripts/application.js +46 -1
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +192 -0
  64. data/web/assets/javascripts/dashboard.js +11 -250
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +4 -0
  67. data/web/assets/stylesheets/application-rtl.css +10 -89
  68. data/web/assets/stylesheets/application.css +98 -295
  69. data/web/locales/ar.yml +70 -70
  70. data/web/locales/cs.yml +62 -62
  71. data/web/locales/da.yml +60 -53
  72. data/web/locales/de.yml +65 -65
  73. data/web/locales/el.yml +43 -24
  74. data/web/locales/en.yml +83 -69
  75. data/web/locales/es.yml +68 -68
  76. data/web/locales/fa.yml +65 -65
  77. data/web/locales/fr.yml +80 -67
  78. data/web/locales/gd.yml +98 -0
  79. data/web/locales/he.yml +65 -64
  80. data/web/locales/hi.yml +59 -59
  81. data/web/locales/it.yml +85 -54
  82. data/web/locales/ja.yml +72 -68
  83. data/web/locales/ko.yml +52 -52
  84. data/web/locales/lt.yml +66 -66
  85. data/web/locales/nb.yml +61 -61
  86. data/web/locales/nl.yml +52 -52
  87. data/web/locales/pl.yml +45 -45
  88. data/web/locales/pt-br.yml +78 -69
  89. data/web/locales/pt.yml +51 -51
  90. data/web/locales/ru.yml +67 -66
  91. data/web/locales/sv.yml +53 -53
  92. data/web/locales/ta.yml +60 -60
  93. data/web/locales/tr.yml +100 -0
  94. data/web/locales/uk.yml +85 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +67 -67
  97. data/web/locales/zh-cn.yml +42 -16
  98. data/web/locales/zh-tw.yml +41 -8
  99. data/web/views/_footer.erb +17 -2
  100. data/web/views/_job_info.erb +18 -2
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +1 -1
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +1 -1
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +49 -33
  107. data/web/views/dashboard.erb +28 -6
  108. data/web/views/filtering.erb +6 -0
  109. data/web/views/layout.erb +6 -6
  110. data/web/views/metrics.erb +90 -0
  111. data/web/views/metrics_for_job.erb +59 -0
  112. data/web/views/morgue.erb +5 -9
  113. data/web/views/queue.erb +15 -15
  114. data/web/views/queues.erb +9 -3
  115. data/web/views/retries.erb +5 -9
  116. data/web/views/scheduled.erb +12 -13
  117. metadata +61 -26
  118. data/lib/sidekiq/.DS_Store +0 -0
  119. data/lib/sidekiq/delay.rb +0 -43
  120. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  121. data/lib/sidekiq/extensions/active_record.rb +0 -43
  122. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  123. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  124. data/lib/sidekiq/worker.rb +0 -367
  125. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -3,7 +3,7 @@
3
3
  module Sidekiq
4
4
  module Paginator
5
5
  def page(key, pageidx = 1, page_size = 25, opts = nil)
6
- current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
6
+ current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
7
7
  pageidx = current_page - 1
8
8
  total_size = 0
9
9
  items = []
@@ -19,9 +19,9 @@ module Sidekiq
19
19
  total_size, items = conn.multi { |transaction|
20
20
  transaction.zcard(key)
21
21
  if rev
22
- transaction.zrevrange(key, starting, ending, withscores: true)
22
+ transaction.zrange(key, starting, ending, "REV", "withscores")
23
23
  else
24
- transaction.zrange(key, starting, ending, withscores: true)
24
+ transaction.zrange(key, starting, ending, "withscores")
25
25
  end
26
26
  }
27
27
  [current_page, total_size, items]
@@ -43,5 +43,13 @@ module Sidekiq
43
43
  end
44
44
  end
45
45
  end
46
+
47
+ def page_items(items, pageidx = 1, page_size = 25)
48
+ current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
49
+ pageidx = current_page - 1
50
+ starting = pageidx * page_size
51
+ items = items.to_a
52
+ [current_page, items.size, items[starting, page_size]]
53
+ end
46
54
  end
47
55
  end
@@ -26,18 +26,18 @@ module Sidekiq
26
26
 
27
27
  attr_reader :thread
28
28
  attr_reader :job
29
+ attr_reader :capsule
29
30
 
30
- def initialize(options, &block)
31
+ def initialize(capsule, &block)
32
+ @config = @capsule = capsule
31
33
  @callback = block
32
34
  @down = false
33
35
  @done = false
34
36
  @job = nil
35
37
  @thread = nil
36
- @config = options
37
- @strategy = options[:fetch]
38
- @reloader = options[:reloader] || proc { |&block| block.call }
39
- @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
40
- @retrier = Sidekiq::JobRetry.new(options)
38
+ @reloader = Sidekiq.default_configuration[:reloader]
39
+ @job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(capsule.config)
40
+ @retrier = Sidekiq::JobRetry.new(capsule)
41
41
  end
42
42
 
43
43
  def terminate(wait = false)
@@ -58,13 +58,21 @@ module Sidekiq
58
58
  @thread.value if wait
59
59
  end
60
60
 
61
+ def stopping?
62
+ @done
63
+ end
64
+
61
65
  def start
62
- @thread ||= safe_thread("processor", &method(:run))
66
+ @thread ||= safe_thread("#{config.name}/processor", &method(:run))
63
67
  end
64
68
 
65
69
  private unless $TESTING
66
70
 
67
71
  def run
72
+ # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
73
+ # instead of the global pool in +Sidekiq::Config#redis_pool+.
74
+ Thread.current[:sidekiq_capsule] = @capsule
75
+
68
76
  process_one until @done
69
77
  @callback.call(self)
70
78
  rescue Sidekiq::Shutdown
@@ -80,7 +88,7 @@ module Sidekiq
80
88
  end
81
89
 
82
90
  def get_one
83
- uow = @strategy.retrieve_work
91
+ uow = capsule.fetcher.retrieve_work
84
92
  if @down
85
93
  logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
86
94
  @down = nil
@@ -129,11 +137,12 @@ module Sidekiq
129
137
  # the Reloader. It handles code loading, db connection management, etc.
130
138
  # Effectively this block denotes a "unit of work" to Rails.
131
139
  @reloader.call do
132
- klass = constantize(job_hash["class"])
133
- inst = klass.new
134
- inst.jid = job_hash["jid"]
135
- @retrier.local(inst, jobstr, queue) do
136
- yield inst
140
+ klass = Object.const_get(job_hash["class"])
141
+ instance = klass.new
142
+ instance.jid = job_hash["jid"]
143
+ instance._context = self
144
+ @retrier.local(instance, jobstr, queue) do
145
+ yield instance
137
146
  end
138
147
  end
139
148
  end
@@ -142,6 +151,11 @@ module Sidekiq
142
151
  end
143
152
  end
144
153
 
154
+ IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
155
+ private_constant :IGNORE_SHUTDOWN_INTERRUPTS
156
+ ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
157
+ private_constant :ALLOW_SHUTDOWN_INTERRUPTS
158
+
145
159
  def process(uow)
146
160
  jobstr = uow.job
147
161
  queue = uow.queue_name
@@ -152,48 +166,58 @@ module Sidekiq
152
166
  job_hash = Sidekiq.load_json(jobstr)
153
167
  rescue => ex
154
168
  handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
155
- # we can't notify because the job isn't a valid hash payload.
156
- DeadSet.new.kill(jobstr, notify_failure: false)
169
+ now = Time.now.to_f
170
+ redis do |conn|
171
+ conn.multi do |xa|
172
+ xa.zadd("dead", now.to_s, jobstr)
173
+ xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
174
+ xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
175
+ end
176
+ end
157
177
  return uow.acknowledge
158
178
  end
159
179
 
160
180
  ack = false
161
- begin
162
- dispatch(job_hash, queue, jobstr) do |inst|
163
- @config.server_middleware.invoke(inst, job_hash, queue) do
164
- execute_job(inst, job_hash["args"])
181
+ Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
182
+ Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
183
+ dispatch(job_hash, queue, jobstr) do |instance|
184
+ config.server_middleware.invoke(instance, job_hash, queue) do
185
+ execute_job(instance, job_hash["args"])
186
+ end
165
187
  end
188
+ ack = true
189
+ rescue Sidekiq::Shutdown
190
+ # Had to force kill this job because it didn't finish
191
+ # within the timeout. Don't acknowledge the work since
192
+ # we didn't properly finish it.
193
+ rescue Sidekiq::JobRetry::Skip => s
194
+ # Skip means we handled this error elsewhere. We don't
195
+ # need to log or report the error.
196
+ ack = true
197
+ raise s
198
+ rescue Sidekiq::JobRetry::Handled => h
199
+ # this is the common case: job raised error and Sidekiq::JobRetry::Handled
200
+ # signals that we created a retry successfully. We can acknowledge the job.
201
+ ack = true
202
+ e = h.cause || h
203
+ handle_exception(e, {context: "Job raised exception", job: job_hash})
204
+ raise e
205
+ rescue Exception => ex
206
+ # Unexpected error! This is very bad and indicates an exception that got past
207
+ # the retry subsystem (e.g. network partition). We won't acknowledge the job
208
+ # so it can be rescued when using Sidekiq Pro.
209
+ handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
210
+ raise ex
166
211
  end
167
- ack = true
168
- rescue Sidekiq::Shutdown
169
- # Had to force kill this job because it didn't finish
170
- # within the timeout. Don't acknowledge the work since
171
- # we didn't properly finish it.
172
- rescue Sidekiq::JobRetry::Handled => h
173
- # this is the common case: job raised error and Sidekiq::JobRetry::Handled
174
- # signals that we created a retry successfully. We can acknowlege the job.
175
- ack = true
176
- e = h.cause || h
177
- handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
178
- raise e
179
- rescue Exception => ex
180
- # Unexpected error! This is very bad and indicates an exception that got past
181
- # the retry subsystem (e.g. network partition). We won't acknowledge the job
182
- # so it can be rescued when using Sidekiq Pro.
183
- handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
184
- raise ex
185
212
  ensure
186
213
  if ack
187
- # We don't want a shutdown signal to interrupt job acknowledgment.
188
- Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
189
- uow.acknowledge
190
- end
214
+ uow.acknowledge
191
215
  end
192
216
  end
193
217
  end
194
218
 
195
- def execute_job(inst, cloned_args)
196
- inst.perform(*cloned_args)
219
+ def execute_job(instance, cloned_args)
220
+ instance.perform(*cloned_args)
197
221
  end
198
222
 
199
223
  # Ruby doesn't provide atomic counters out of the box so we'll
@@ -263,18 +287,5 @@ module Sidekiq
263
287
  PROCESSED.incr
264
288
  end
265
289
  end
266
-
267
- def constantize(str)
268
- return Object.const_get(str) unless str.include?("::")
269
-
270
- names = str.split("::")
271
- names.shift if names.empty? || names.first.empty?
272
-
273
- names.inject(Object) do |constant, name|
274
- # the false flag limits search for name to under the constant namespace
275
- # which mimics Rails' behaviour
276
- constant.const_get(name, false)
277
- end
278
- end
279
290
  end
280
291
  end
data/lib/sidekiq/rails.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq/job"
4
+ require "rails"
4
5
 
5
6
  module Sidekiq
6
7
  class Rails < ::Rails::Engine
@@ -10,7 +11,8 @@ module Sidekiq
10
11
  end
11
12
 
12
13
  def call
13
- @app.reloader.wrap do
14
+ params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
15
+ @app.reloader.wrap(**params) do
14
16
  yield
15
17
  end
16
18
  end
@@ -18,11 +20,15 @@ module Sidekiq
18
20
  def inspect
19
21
  "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
20
22
  end
23
+
24
+ def to_hash
25
+ {app: @app.class.name}
26
+ end
21
27
  end
22
28
 
23
29
  # By including the Options module, we allow AJs to directly control sidekiq features
24
30
  # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
25
- # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
31
+ # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
26
32
  # manually retried, don't automatically die, etc.
27
33
  #
28
34
  # class SomeJob < ActiveJob::Base
@@ -33,28 +39,18 @@ module Sidekiq
33
39
  # end
34
40
  initializer "sidekiq.active_job_integration" do
35
41
  ActiveSupport.on_load(:active_job) do
42
+ require "active_job/queue_adapters/sidekiq_adapter"
43
+
36
44
  include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
37
45
  end
38
46
  end
39
47
 
40
- initializer "sidekiq.rails_logger" do
48
+ initializer "sidekiq.backtrace_cleaner" do
41
49
  Sidekiq.configure_server do |config|
42
- # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
43
- # it will appear in the Sidekiq console with all of the job context. See #5021 and
44
- # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
45
- unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
46
- ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
47
- end
50
+ config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
48
51
  end
49
52
  end
50
53
 
51
- config.before_configuration do
52
- dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
53
- dep.deprecate_methods(Sidekiq.singleton_class,
54
- default_worker_options: :default_job_options,
55
- "default_worker_options=": :default_job_options=)
56
- end
57
-
58
54
  # This hook happens after all initializers are run, just before returning
59
55
  # from config/environment.rb back to sidekiq/cli.rb.
60
56
  #
@@ -62,6 +58,16 @@ module Sidekiq
62
58
  config.after_initialize do
63
59
  Sidekiq.configure_server do |config|
64
60
  config[:reloader] = Sidekiq::Rails::Reloader.new
61
+
62
+ # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
63
+ # it will appear in the Sidekiq console with all of the job context.
64
+ unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
65
+ if ::Rails.logger.respond_to?(:broadcast_to)
66
+ ::Rails.logger.broadcast_to(config.logger)
67
+ else
68
+ ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
69
+ end
70
+ end
65
71
  end
66
72
  end
67
73
  end
@@ -1,15 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "connection_pool"
3
+ require "set"
4
4
  require "redis_client"
5
5
  require "redis_client/decorator"
6
- require "uri"
7
6
 
8
7
  module Sidekiq
9
8
  class RedisClientAdapter
10
9
  BaseError = RedisClient::Error
11
10
  CommandError = RedisClient::CommandError
12
11
 
12
+ # You can add/remove items or clear the whole thing if you don't want deprecation warnings.
13
+ DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
14
+
13
15
  module CompatMethods
14
16
  def info
15
17
  @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
@@ -19,30 +21,28 @@ module Sidekiq
19
21
  @client.call("EVALSHA", sha, keys.size, *keys, *argv)
20
22
  end
21
23
 
22
- def brpoplpush(*args)
23
- @client.blocking_call(false, "BRPOPLPUSH", *args)
24
- end
25
-
26
- def brpop(*args)
27
- @client.blocking_call(false, "BRPOP", *args)
28
- end
29
-
30
- def set(*args)
31
- @client.call("SET", *args) { |r| r == "OK" }
32
- end
33
- ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
34
-
35
- def sismember(*args)
36
- @client.call("SISMEMBER", *args) { |c| c > 0 }
37
- end
24
+ # this is the set of Redis commands used by Sidekiq. Not guaranteed
25
+ # to be comprehensive, we use this as a performance enhancement to
26
+ # avoid calling method_missing on most commands
27
+ USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
28
+ get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
29
+ lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
30
+ publish rpop rpush sadd scard script set sismember smembers
31
+ srem ttl type unlink zadd zcard zincrby zrange zrem
32
+ zremrangebyrank zremrangebyscore]
38
33
 
39
- def exists?(key)
40
- @client.call("EXISTS", key) { |c| c > 0 }
34
+ USED_COMMANDS.each do |name|
35
+ define_method(name) do |*args, **kwargs|
36
+ @client.call(name, *args, **kwargs)
37
+ end
41
38
  end
42
39
 
43
40
  private
44
41
 
42
+ # this allows us to use methods like `conn.hmset(...)` instead of having to use
43
+ # redis-client's native `conn.call("hmset", ...)`
45
44
  def method_missing(*args, &block)
45
+ warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
46
46
  @client.call(*args, *block)
47
47
  end
48
48
  ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
@@ -55,47 +55,8 @@ module Sidekiq
55
55
  CompatClient = RedisClient::Decorator.create(CompatMethods)
56
56
 
57
57
  class CompatClient
58
- %i[scan sscan zscan hscan].each do |method|
59
- alias_method :"#{method}_each", method
60
- undef_method method
61
- end
62
-
63
- def disconnect!
64
- @client.close
65
- end
66
-
67
- def connection
68
- {id: @client.id}
69
- end
70
-
71
- def redis
72
- self
73
- end
74
-
75
- def _client
76
- @client
77
- end
78
-
79
- def message
80
- yield nil, @queue.pop
81
- end
82
-
83
- # NB: this method does not return
84
- def subscribe(chan)
85
- @queue = ::Queue.new
86
-
87
- pubsub = @client.pubsub
88
- pubsub.call("subscribe", chan)
89
-
90
- loop do
91
- evt = pubsub.next_event
92
- next if evt.nil?
93
- next unless evt[0] == "message" && evt[1] == chan
94
-
95
- (_, _, msg) = evt
96
- @queue << msg
97
- yield self
98
- end
58
+ def config
59
+ @client.config
99
60
  end
100
61
  end
101
62
 
@@ -103,6 +64,13 @@ module Sidekiq
103
64
  opts = client_opts(options)
104
65
  @config = if opts.key?(:sentinels)
105
66
  RedisClient.sentinel(**opts)
67
+ elsif opts.key?(:nodes)
68
+ # Sidekiq does not support Redis clustering but Sidekiq Enterprise's
69
+ # rate limiters are cluster-safe so we can scale to millions
70
+ # of rate limiters using a Redis cluster. This requires the
71
+ # `redis-cluster-client` gem.
72
+ # Sidekiq::Limiter.redis = { nodes: [...] }
73
+ RedisClient.cluster(**opts)
106
74
  else
107
75
  RedisClient.config(**opts)
108
76
  end
@@ -118,9 +86,7 @@ module Sidekiq
118
86
  opts = options.dup
119
87
 
120
88
  if opts[:namespace]
121
- Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
122
- "Either use the redis adapter or remove the namespace.")
123
- Kernel.exit(-127)
89
+ raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
124
90
  end
125
91
 
126
92
  opts.delete(:size)
@@ -131,13 +97,9 @@ module Sidekiq
131
97
  opts.delete(:network_timeout)
132
98
  end
133
99
 
134
- if opts[:driver]
135
- opts[:driver] = opts[:driver].to_sym
136
- end
137
-
138
100
  opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
139
101
  opts[:role] = opts[:role].to_sym if opts.key?(:role)
140
- opts.delete(:url) if opts.key?(:sentinels)
102
+ opts[:driver] = opts[:driver].to_sym if opts.key?(:driver)
141
103
 
142
104
  # Issue #3303, redis-rb will silently retry an operation.
143
105
  # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
@@ -150,5 +112,3 @@ module Sidekiq
150
112
  end
151
113
  end
152
114
  end
153
-
154
- Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
@@ -1,130 +1,57 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "connection_pool"
4
- require "redis"
5
4
  require "uri"
5
+ require "sidekiq/redis_client_adapter"
6
6
 
7
7
  module Sidekiq
8
8
  module RedisConnection
9
- class RedisAdapter
10
- BaseError = Redis::BaseError
11
- CommandError = Redis::CommandError
12
-
13
- def initialize(options)
14
- warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
15
- @options = options
16
- end
17
-
18
- def new_client
19
- namespace = @options[:namespace]
20
-
21
- client = Redis.new client_opts(@options)
22
- if namespace
23
- begin
24
- require "redis/namespace"
25
- Redis::Namespace.new(namespace, redis: client)
26
- rescue LoadError
27
- Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
28
- "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
29
- exit(-127)
30
- end
31
- else
32
- client
33
- end
34
- end
35
-
36
- private
37
-
38
- def client_opts(options)
39
- opts = options.dup
40
- if opts[:namespace]
41
- opts.delete(:namespace)
42
- end
43
-
44
- if opts[:network_timeout]
45
- opts[:timeout] = opts[:network_timeout]
46
- opts.delete(:network_timeout)
47
- end
48
-
49
- opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
50
-
51
- # Issue #3303, redis-rb will silently retry an operation.
52
- # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
53
- # is performed twice but I believe this is much, much rarer
54
- # than the reconnect silently fixing a problem; we keep it
55
- # on by default.
56
- opts[:reconnect_attempts] ||= 1
57
-
58
- opts
59
- end
60
- end
61
-
62
- @adapter = RedisAdapter
63
-
64
9
  class << self
65
- attr_reader :adapter
66
-
67
- # RedisConnection.adapter = :redis
68
- # RedisConnection.adapter = :redis_client
69
- def adapter=(adapter)
70
- raise "no" if adapter == self
71
- result = case adapter
72
- when :redis
73
- RedisAdapter
74
- when Class
75
- adapter
76
- else
77
- require "sidekiq/#{adapter}_adapter"
78
- nil
79
- end
80
- @adapter = result if result
81
- end
82
-
83
10
  def create(options = {})
84
- symbolized_options = options.transform_keys(&:to_sym)
11
+ symbolized_options = deep_symbolize_keys(options)
12
+ symbolized_options[:url] ||= determine_redis_provider
85
13
 
86
- if !symbolized_options[:url] && (u = determine_redis_provider)
87
- symbolized_options[:url] = u
88
- end
14
+ logger = symbolized_options.delete(:logger)
15
+ logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
89
16
 
90
- size = if symbolized_options[:size]
91
- symbolized_options[:size]
92
- elsif Sidekiq.server?
93
- # Give ourselves plenty of connections. pool is lazy
94
- # so we won't create them until we need them.
95
- Sidekiq[:concurrency] + 5
96
- elsif ENV["RAILS_MAX_THREADS"]
97
- Integer(ENV["RAILS_MAX_THREADS"])
98
- else
99
- 5
100
- end
17
+ raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
18
+
19
+ safe = !!symbolized_options.delete(:cluster_safe)
20
+ raise ":nodes not allowed, Sidekiq is not safe to run on Redis Cluster" if !safe && symbolized_options.key?(:nodes)
101
21
 
102
- verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
22
+ size = symbolized_options.delete(:size) || 5
23
+ pool_timeout = symbolized_options.delete(:pool_timeout) || 1
24
+ pool_name = symbolized_options.delete(:pool_name)
103
25
 
104
- pool_timeout = symbolized_options[:pool_timeout] || 1
105
- log_info(symbolized_options)
26
+ # Default timeout in redis-client is 1 second, which can be too aggressive
27
+ # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
28
+ # it can be easy to get the occasional ReadTimeoutError. You can still provide
29
+ # a smaller timeout explicitly:
30
+ # config.redis = { url: "...", timeout: 1 }
31
+ symbolized_options[:timeout] ||= 3
106
32
 
107
- redis_config = adapter.new(symbolized_options)
108
- ConnectionPool.new(timeout: pool_timeout, size: size) do
33
+ redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
34
+ ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
109
35
  redis_config.new_client
110
36
  end
111
37
  end
112
38
 
113
39
  private
114
40
 
115
- # Sidekiq needs many concurrent Redis connections.
116
- #
117
- # We need a connection for each Processor.
118
- # We need a connection for Pro's real-time change listener
119
- # We need a connection to various features to call Redis every few seconds:
120
- # - the process heartbeat.
121
- # - enterprise's leader election
122
- # - enterprise's cron support
123
- def verify_sizing(size, concurrency)
124
- raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
41
+ def deep_symbolize_keys(object)
42
+ case object
43
+ when Hash
44
+ object.each_with_object({}) do |(key, value), result|
45
+ result[key.to_sym] = deep_symbolize_keys(value)
46
+ end
47
+ when Array
48
+ object.map { |e| deep_symbolize_keys(e) }
49
+ else
50
+ object
51
+ end
125
52
  end
126
53
 
127
- def log_info(options)
54
+ def scrub(options)
128
55
  redacted = "REDACTED"
129
56
 
130
57
  # Deep clone so we can muck with these options all we want and exclude
@@ -136,17 +63,19 @@ module Sidekiq
136
63
  uri.password = redacted
137
64
  scrubbed_options[:url] = uri.to_s
138
65
  end
139
- if scrubbed_options[:password]
140
- scrubbed_options[:password] = redacted
141
- end
66
+ scrubbed_options[:password] = redacted if scrubbed_options[:password]
67
+ scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
142
68
  scrubbed_options[:sentinels]&.each do |sentinel|
143
- sentinel[:password] = redacted if sentinel[:password]
144
- end
145
- if Sidekiq.server?
146
- Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
147
- else
148
- Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
69
+ if sentinel.is_a?(String)
70
+ if (uri = URI(sentinel)) && uri.password
71
+ uri.password = redacted
72
+ sentinel.replace(uri.to_s)
73
+ end
74
+ elsif sentinel[:password]
75
+ sentinel[:password] = redacted
76
+ end
149
77
  end
78
+ scrubbed_options
150
79
  end
151
80
 
152
81
  def determine_redis_provider
@@ -168,9 +97,7 @@ module Sidekiq
168
97
  EOM
169
98
  end
170
99
 
171
- ENV[
172
- p || "REDIS_URL"
173
- ]
100
+ ENV[p.to_s] || ENV["REDIS_URL"]
174
101
  end
175
102
  end
176
103
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "forwardable"
2
4
 
3
5
  module Sidekiq