sidekiq 6.5.1 → 7.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +376 -12
- data/README.md +43 -35
- data/bin/multi_queue_bench +271 -0
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +213 -118
- data/bin/sidekiqmon +3 -0
- data/lib/active_job/queue_adapters/sidekiq_adapter.rb +88 -0
- data/lib/generators/sidekiq/job_generator.rb +2 -0
- data/lib/sidekiq/api.rb +378 -173
- data/lib/sidekiq/capsule.rb +132 -0
- data/lib/sidekiq/cli.rb +61 -63
- data/lib/sidekiq/client.rb +89 -40
- data/lib/sidekiq/component.rb +6 -2
- data/lib/sidekiq/config.rb +305 -0
- data/lib/sidekiq/deploy.rb +64 -0
- data/lib/sidekiq/embedded.rb +63 -0
- data/lib/sidekiq/fetch.rb +11 -14
- data/lib/sidekiq/iterable_job.rb +55 -0
- data/lib/sidekiq/job/interrupt_handler.rb +24 -0
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
- data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
- data/lib/sidekiq/job/iterable.rb +294 -0
- data/lib/sidekiq/job.rb +382 -10
- data/lib/sidekiq/job_logger.rb +8 -7
- data/lib/sidekiq/job_retry.rb +89 -46
- data/lib/sidekiq/job_util.rb +53 -15
- data/lib/sidekiq/launcher.rb +77 -69
- data/lib/sidekiq/logger.rb +2 -27
- data/lib/sidekiq/manager.rb +9 -11
- data/lib/sidekiq/metrics/query.rb +158 -0
- data/lib/sidekiq/metrics/shared.rb +106 -0
- data/lib/sidekiq/metrics/tracking.rb +148 -0
- data/lib/sidekiq/middleware/chain.rb +84 -48
- data/lib/sidekiq/middleware/current_attributes.rb +87 -20
- data/lib/sidekiq/middleware/modules.rb +2 -0
- data/lib/sidekiq/monitor.rb +19 -5
- data/lib/sidekiq/paginator.rb +11 -3
- data/lib/sidekiq/processor.rb +67 -56
- data/lib/sidekiq/rails.rb +22 -16
- data/lib/sidekiq/redis_client_adapter.rb +31 -71
- data/lib/sidekiq/redis_connection.rb +44 -117
- data/lib/sidekiq/ring_buffer.rb +2 -0
- data/lib/sidekiq/scheduled.rb +62 -35
- data/lib/sidekiq/systemd.rb +2 -0
- data/lib/sidekiq/testing.rb +37 -46
- data/lib/sidekiq/transaction_aware_client.rb +11 -5
- data/lib/sidekiq/version.rb +6 -1
- data/lib/sidekiq/web/action.rb +15 -5
- data/lib/sidekiq/web/application.rb +94 -24
- data/lib/sidekiq/web/csrf_protection.rb +10 -7
- data/lib/sidekiq/web/helpers.rb +118 -45
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +67 -15
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +78 -266
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +46 -1
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +192 -0
- data/web/assets/javascripts/dashboard.js +11 -250
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +10 -89
- data/web/assets/stylesheets/application.css +98 -295
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +83 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +80 -67
- data/web/locales/gd.yml +98 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +85 -54
- data/web/locales/ja.yml +72 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +78 -69
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/tr.yml +100 -0
- data/web/locales/uk.yml +85 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +42 -16
- data/web/locales/zh-tw.yml +41 -8
- data/web/views/_footer.erb +17 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +49 -33
- data/web/views/dashboard.erb +28 -6
- data/web/views/filtering.erb +6 -0
- data/web/views/layout.erb +6 -6
- data/web/views/metrics.erb +90 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +9 -3
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +61 -26
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/worker.rb +0 -367
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/paginator.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Sidekiq
|
4
4
|
module Paginator
|
5
5
|
def page(key, pageidx = 1, page_size = 25, opts = nil)
|
6
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
6
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
7
7
|
pageidx = current_page - 1
|
8
8
|
total_size = 0
|
9
9
|
items = []
|
@@ -19,9 +19,9 @@ module Sidekiq
|
|
19
19
|
total_size, items = conn.multi { |transaction|
|
20
20
|
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
transaction.
|
22
|
+
transaction.zrange(key, starting, ending, "REV", "withscores")
|
23
23
|
else
|
24
|
-
transaction.zrange(key, starting, ending, withscores
|
24
|
+
transaction.zrange(key, starting, ending, "withscores")
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
@@ -43,5 +43,13 @@ module Sidekiq
|
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
46
|
+
|
47
|
+
def page_items(items, pageidx = 1, page_size = 25)
|
48
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
49
|
+
pageidx = current_page - 1
|
50
|
+
starting = pageidx * page_size
|
51
|
+
items = items.to_a
|
52
|
+
[current_page, items.size, items[starting, page_size]]
|
53
|
+
end
|
46
54
|
end
|
47
55
|
end
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -26,18 +26,18 @@ module Sidekiq
|
|
26
26
|
|
27
27
|
attr_reader :thread
|
28
28
|
attr_reader :job
|
29
|
+
attr_reader :capsule
|
29
30
|
|
30
|
-
def initialize(
|
31
|
+
def initialize(capsule, &block)
|
32
|
+
@config = @capsule = capsule
|
31
33
|
@callback = block
|
32
34
|
@down = false
|
33
35
|
@done = false
|
34
36
|
@job = nil
|
35
37
|
@thread = nil
|
36
|
-
@
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
-
@retrier = Sidekiq::JobRetry.new(options)
|
38
|
+
@reloader = Sidekiq.default_configuration[:reloader]
|
39
|
+
@job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(capsule.config)
|
40
|
+
@retrier = Sidekiq::JobRetry.new(capsule)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -58,13 +58,21 @@ module Sidekiq
|
|
58
58
|
@thread.value if wait
|
59
59
|
end
|
60
60
|
|
61
|
+
def stopping?
|
62
|
+
@done
|
63
|
+
end
|
64
|
+
|
61
65
|
def start
|
62
|
-
@thread ||= safe_thread("processor", &method(:run))
|
66
|
+
@thread ||= safe_thread("#{config.name}/processor", &method(:run))
|
63
67
|
end
|
64
68
|
|
65
69
|
private unless $TESTING
|
66
70
|
|
67
71
|
def run
|
72
|
+
# By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
|
73
|
+
# instead of the global pool in +Sidekiq::Config#redis_pool+.
|
74
|
+
Thread.current[:sidekiq_capsule] = @capsule
|
75
|
+
|
68
76
|
process_one until @done
|
69
77
|
@callback.call(self)
|
70
78
|
rescue Sidekiq::Shutdown
|
@@ -80,7 +88,7 @@ module Sidekiq
|
|
80
88
|
end
|
81
89
|
|
82
90
|
def get_one
|
83
|
-
uow =
|
91
|
+
uow = capsule.fetcher.retrieve_work
|
84
92
|
if @down
|
85
93
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
94
|
@down = nil
|
@@ -129,11 +137,12 @@ module Sidekiq
|
|
129
137
|
# the Reloader. It handles code loading, db connection management, etc.
|
130
138
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
139
|
@reloader.call do
|
132
|
-
klass =
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
140
|
+
klass = Object.const_get(job_hash["class"])
|
141
|
+
instance = klass.new
|
142
|
+
instance.jid = job_hash["jid"]
|
143
|
+
instance._context = self
|
144
|
+
@retrier.local(instance, jobstr, queue) do
|
145
|
+
yield instance
|
137
146
|
end
|
138
147
|
end
|
139
148
|
end
|
@@ -142,6 +151,11 @@ module Sidekiq
|
|
142
151
|
end
|
143
152
|
end
|
144
153
|
|
154
|
+
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
155
|
+
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
156
|
+
ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
|
157
|
+
private_constant :ALLOW_SHUTDOWN_INTERRUPTS
|
158
|
+
|
145
159
|
def process(uow)
|
146
160
|
jobstr = uow.job
|
147
161
|
queue = uow.queue_name
|
@@ -152,48 +166,58 @@ module Sidekiq
|
|
152
166
|
job_hash = Sidekiq.load_json(jobstr)
|
153
167
|
rescue => ex
|
154
168
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
-
|
156
|
-
|
169
|
+
now = Time.now.to_f
|
170
|
+
redis do |conn|
|
171
|
+
conn.multi do |xa|
|
172
|
+
xa.zadd("dead", now.to_s, jobstr)
|
173
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
174
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
175
|
+
end
|
176
|
+
end
|
157
177
|
return uow.acknowledge
|
158
178
|
end
|
159
179
|
|
160
180
|
ack = false
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
181
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
182
|
+
Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
|
183
|
+
dispatch(job_hash, queue, jobstr) do |instance|
|
184
|
+
config.server_middleware.invoke(instance, job_hash, queue) do
|
185
|
+
execute_job(instance, job_hash["args"])
|
186
|
+
end
|
165
187
|
end
|
188
|
+
ack = true
|
189
|
+
rescue Sidekiq::Shutdown
|
190
|
+
# Had to force kill this job because it didn't finish
|
191
|
+
# within the timeout. Don't acknowledge the work since
|
192
|
+
# we didn't properly finish it.
|
193
|
+
rescue Sidekiq::JobRetry::Skip => s
|
194
|
+
# Skip means we handled this error elsewhere. We don't
|
195
|
+
# need to log or report the error.
|
196
|
+
ack = true
|
197
|
+
raise s
|
198
|
+
rescue Sidekiq::JobRetry::Handled => h
|
199
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
200
|
+
# signals that we created a retry successfully. We can acknowledge the job.
|
201
|
+
ack = true
|
202
|
+
e = h.cause || h
|
203
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
204
|
+
raise e
|
205
|
+
rescue Exception => ex
|
206
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
207
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
208
|
+
# so it can be rescued when using Sidekiq Pro.
|
209
|
+
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
210
|
+
raise ex
|
166
211
|
end
|
167
|
-
ack = true
|
168
|
-
rescue Sidekiq::Shutdown
|
169
|
-
# Had to force kill this job because it didn't finish
|
170
|
-
# within the timeout. Don't acknowledge the work since
|
171
|
-
# we didn't properly finish it.
|
172
|
-
rescue Sidekiq::JobRetry::Handled => h
|
173
|
-
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
174
|
-
# signals that we created a retry successfully. We can acknowlege the job.
|
175
|
-
ack = true
|
176
|
-
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
|
178
|
-
raise e
|
179
|
-
rescue Exception => ex
|
180
|
-
# Unexpected error! This is very bad and indicates an exception that got past
|
181
|
-
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
182
|
-
# so it can be rescued when using Sidekiq Pro.
|
183
|
-
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
184
|
-
raise ex
|
185
212
|
ensure
|
186
213
|
if ack
|
187
|
-
|
188
|
-
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
uow.acknowledge
|
190
|
-
end
|
214
|
+
uow.acknowledge
|
191
215
|
end
|
192
216
|
end
|
193
217
|
end
|
194
218
|
|
195
|
-
def execute_job(
|
196
|
-
|
219
|
+
def execute_job(instance, cloned_args)
|
220
|
+
instance.perform(*cloned_args)
|
197
221
|
end
|
198
222
|
|
199
223
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -263,18 +287,5 @@ module Sidekiq
|
|
263
287
|
PROCESSED.incr
|
264
288
|
end
|
265
289
|
end
|
266
|
-
|
267
|
-
def constantize(str)
|
268
|
-
return Object.const_get(str) unless str.include?("::")
|
269
|
-
|
270
|
-
names = str.split("::")
|
271
|
-
names.shift if names.empty? || names.first.empty?
|
272
|
-
|
273
|
-
names.inject(Object) do |constant, name|
|
274
|
-
# the false flag limits search for name to under the constant namespace
|
275
|
-
# which mimics Rails' behaviour
|
276
|
-
constant.const_get(name, false)
|
277
|
-
end
|
278
|
-
end
|
279
290
|
end
|
280
291
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq/job"
|
4
|
+
require "rails"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Rails < ::Rails::Engine
|
@@ -10,7 +11,8 @@ module Sidekiq
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def call
|
13
|
-
|
14
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
15
|
+
@app.reloader.wrap(**params) do
|
14
16
|
yield
|
15
17
|
end
|
16
18
|
end
|
@@ -18,11 +20,15 @@ module Sidekiq
|
|
18
20
|
def inspect
|
19
21
|
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
22
|
end
|
23
|
+
|
24
|
+
def to_hash
|
25
|
+
{app: @app.class.name}
|
26
|
+
end
|
21
27
|
end
|
22
28
|
|
23
29
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
30
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
-
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
31
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
26
32
|
# manually retried, don't automatically die, etc.
|
27
33
|
#
|
28
34
|
# class SomeJob < ActiveJob::Base
|
@@ -33,28 +39,18 @@ module Sidekiq
|
|
33
39
|
# end
|
34
40
|
initializer "sidekiq.active_job_integration" do
|
35
41
|
ActiveSupport.on_load(:active_job) do
|
42
|
+
require "active_job/queue_adapters/sidekiq_adapter"
|
43
|
+
|
36
44
|
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
45
|
end
|
38
46
|
end
|
39
47
|
|
40
|
-
initializer "sidekiq.
|
48
|
+
initializer "sidekiq.backtrace_cleaner" do
|
41
49
|
Sidekiq.configure_server do |config|
|
42
|
-
|
43
|
-
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
-
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
-
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
|
-
end
|
50
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
48
51
|
end
|
49
52
|
end
|
50
53
|
|
51
|
-
config.before_configuration do
|
52
|
-
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
-
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
-
default_worker_options: :default_job_options,
|
55
|
-
"default_worker_options=": :default_job_options=)
|
56
|
-
end
|
57
|
-
|
58
54
|
# This hook happens after all initializers are run, just before returning
|
59
55
|
# from config/environment.rb back to sidekiq/cli.rb.
|
60
56
|
#
|
@@ -62,6 +58,16 @@ module Sidekiq
|
|
62
58
|
config.after_initialize do
|
63
59
|
Sidekiq.configure_server do |config|
|
64
60
|
config[:reloader] = Sidekiq::Rails::Reloader.new
|
61
|
+
|
62
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
63
|
+
# it will appear in the Sidekiq console with all of the job context.
|
64
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
65
|
+
if ::Rails.logger.respond_to?(:broadcast_to)
|
66
|
+
::Rails.logger.broadcast_to(config.logger)
|
67
|
+
else
|
68
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
69
|
+
end
|
70
|
+
end
|
65
71
|
end
|
66
72
|
end
|
67
73
|
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "
|
3
|
+
require "set"
|
4
4
|
require "redis_client"
|
5
5
|
require "redis_client/decorator"
|
6
|
-
require "uri"
|
7
6
|
|
8
7
|
module Sidekiq
|
9
8
|
class RedisClientAdapter
|
10
9
|
BaseError = RedisClient::Error
|
11
10
|
CommandError = RedisClient::CommandError
|
12
11
|
|
12
|
+
# You can add/remove items or clear the whole thing if you don't want deprecation warnings.
|
13
|
+
DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
|
14
|
+
|
13
15
|
module CompatMethods
|
14
16
|
def info
|
15
17
|
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
@@ -19,30 +21,28 @@ module Sidekiq
|
|
19
21
|
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
22
|
end
|
21
23
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
-
end
|
33
|
-
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
-
|
35
|
-
def sismember(*args)
|
36
|
-
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
-
end
|
24
|
+
# this is the set of Redis commands used by Sidekiq. Not guaranteed
|
25
|
+
# to be comprehensive, we use this as a performance enhancement to
|
26
|
+
# avoid calling method_missing on most commands
|
27
|
+
USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
|
28
|
+
get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
|
29
|
+
lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
|
30
|
+
publish rpop rpush sadd scard script set sismember smembers
|
31
|
+
srem ttl type unlink zadd zcard zincrby zrange zrem
|
32
|
+
zremrangebyrank zremrangebyscore]
|
38
33
|
|
39
|
-
|
40
|
-
|
34
|
+
USED_COMMANDS.each do |name|
|
35
|
+
define_method(name) do |*args, **kwargs|
|
36
|
+
@client.call(name, *args, **kwargs)
|
37
|
+
end
|
41
38
|
end
|
42
39
|
|
43
40
|
private
|
44
41
|
|
42
|
+
# this allows us to use methods like `conn.hmset(...)` instead of having to use
|
43
|
+
# redis-client's native `conn.call("hmset", ...)`
|
45
44
|
def method_missing(*args, &block)
|
45
|
+
warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
|
46
46
|
@client.call(*args, *block)
|
47
47
|
end
|
48
48
|
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
@@ -55,47 +55,8 @@ module Sidekiq
|
|
55
55
|
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
56
|
|
57
57
|
class CompatClient
|
58
|
-
|
59
|
-
|
60
|
-
undef_method method
|
61
|
-
end
|
62
|
-
|
63
|
-
def disconnect!
|
64
|
-
@client.close
|
65
|
-
end
|
66
|
-
|
67
|
-
def connection
|
68
|
-
{id: @client.id}
|
69
|
-
end
|
70
|
-
|
71
|
-
def redis
|
72
|
-
self
|
73
|
-
end
|
74
|
-
|
75
|
-
def _client
|
76
|
-
@client
|
77
|
-
end
|
78
|
-
|
79
|
-
def message
|
80
|
-
yield nil, @queue.pop
|
81
|
-
end
|
82
|
-
|
83
|
-
# NB: this method does not return
|
84
|
-
def subscribe(chan)
|
85
|
-
@queue = ::Queue.new
|
86
|
-
|
87
|
-
pubsub = @client.pubsub
|
88
|
-
pubsub.call("subscribe", chan)
|
89
|
-
|
90
|
-
loop do
|
91
|
-
evt = pubsub.next_event
|
92
|
-
next if evt.nil?
|
93
|
-
next unless evt[0] == "message" && evt[1] == chan
|
94
|
-
|
95
|
-
(_, _, msg) = evt
|
96
|
-
@queue << msg
|
97
|
-
yield self
|
98
|
-
end
|
58
|
+
def config
|
59
|
+
@client.config
|
99
60
|
end
|
100
61
|
end
|
101
62
|
|
@@ -103,6 +64,13 @@ module Sidekiq
|
|
103
64
|
opts = client_opts(options)
|
104
65
|
@config = if opts.key?(:sentinels)
|
105
66
|
RedisClient.sentinel(**opts)
|
67
|
+
elsif opts.key?(:nodes)
|
68
|
+
# Sidekiq does not support Redis clustering but Sidekiq Enterprise's
|
69
|
+
# rate limiters are cluster-safe so we can scale to millions
|
70
|
+
# of rate limiters using a Redis cluster. This requires the
|
71
|
+
# `redis-cluster-client` gem.
|
72
|
+
# Sidekiq::Limiter.redis = { nodes: [...] }
|
73
|
+
RedisClient.cluster(**opts)
|
106
74
|
else
|
107
75
|
RedisClient.config(**opts)
|
108
76
|
end
|
@@ -118,9 +86,7 @@ module Sidekiq
|
|
118
86
|
opts = options.dup
|
119
87
|
|
120
88
|
if opts[:namespace]
|
121
|
-
|
122
|
-
"Either use the redis adapter or remove the namespace.")
|
123
|
-
Kernel.exit(-127)
|
89
|
+
raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
|
124
90
|
end
|
125
91
|
|
126
92
|
opts.delete(:size)
|
@@ -131,13 +97,9 @@ module Sidekiq
|
|
131
97
|
opts.delete(:network_timeout)
|
132
98
|
end
|
133
99
|
|
134
|
-
if opts[:driver]
|
135
|
-
opts[:driver] = opts[:driver].to_sym
|
136
|
-
end
|
137
|
-
|
138
100
|
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
101
|
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
-
opts
|
102
|
+
opts[:driver] = opts[:driver].to_sym if opts.key?(:driver)
|
141
103
|
|
142
104
|
# Issue #3303, redis-rb will silently retry an operation.
|
143
105
|
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
@@ -150,5 +112,3 @@ module Sidekiq
|
|
150
112
|
end
|
151
113
|
end
|
152
114
|
end
|
153
|
-
|
154
|
-
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -1,130 +1,57 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "connection_pool"
|
4
|
-
require "redis"
|
5
4
|
require "uri"
|
5
|
+
require "sidekiq/redis_client_adapter"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
module RedisConnection
|
9
|
-
class RedisAdapter
|
10
|
-
BaseError = Redis::BaseError
|
11
|
-
CommandError = Redis::CommandError
|
12
|
-
|
13
|
-
def initialize(options)
|
14
|
-
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
-
@options = options
|
16
|
-
end
|
17
|
-
|
18
|
-
def new_client
|
19
|
-
namespace = @options[:namespace]
|
20
|
-
|
21
|
-
client = Redis.new client_opts(@options)
|
22
|
-
if namespace
|
23
|
-
begin
|
24
|
-
require "redis/namespace"
|
25
|
-
Redis::Namespace.new(namespace, redis: client)
|
26
|
-
rescue LoadError
|
27
|
-
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
28
|
-
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
29
|
-
exit(-127)
|
30
|
-
end
|
31
|
-
else
|
32
|
-
client
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def client_opts(options)
|
39
|
-
opts = options.dup
|
40
|
-
if opts[:namespace]
|
41
|
-
opts.delete(:namespace)
|
42
|
-
end
|
43
|
-
|
44
|
-
if opts[:network_timeout]
|
45
|
-
opts[:timeout] = opts[:network_timeout]
|
46
|
-
opts.delete(:network_timeout)
|
47
|
-
end
|
48
|
-
|
49
|
-
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
50
|
-
|
51
|
-
# Issue #3303, redis-rb will silently retry an operation.
|
52
|
-
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
53
|
-
# is performed twice but I believe this is much, much rarer
|
54
|
-
# than the reconnect silently fixing a problem; we keep it
|
55
|
-
# on by default.
|
56
|
-
opts[:reconnect_attempts] ||= 1
|
57
|
-
|
58
|
-
opts
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
@adapter = RedisAdapter
|
63
|
-
|
64
9
|
class << self
|
65
|
-
attr_reader :adapter
|
66
|
-
|
67
|
-
# RedisConnection.adapter = :redis
|
68
|
-
# RedisConnection.adapter = :redis_client
|
69
|
-
def adapter=(adapter)
|
70
|
-
raise "no" if adapter == self
|
71
|
-
result = case adapter
|
72
|
-
when :redis
|
73
|
-
RedisAdapter
|
74
|
-
when Class
|
75
|
-
adapter
|
76
|
-
else
|
77
|
-
require "sidekiq/#{adapter}_adapter"
|
78
|
-
nil
|
79
|
-
end
|
80
|
-
@adapter = result if result
|
81
|
-
end
|
82
|
-
|
83
10
|
def create(options = {})
|
84
|
-
symbolized_options = options
|
11
|
+
symbolized_options = deep_symbolize_keys(options)
|
12
|
+
symbolized_options[:url] ||= determine_redis_provider
|
85
13
|
|
86
|
-
|
87
|
-
|
88
|
-
end
|
14
|
+
logger = symbolized_options.delete(:logger)
|
15
|
+
logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
|
89
16
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
# so we won't create them until we need them.
|
95
|
-
Sidekiq[:concurrency] + 5
|
96
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
97
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
98
|
-
else
|
99
|
-
5
|
100
|
-
end
|
17
|
+
raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
|
18
|
+
|
19
|
+
safe = !!symbolized_options.delete(:cluster_safe)
|
20
|
+
raise ":nodes not allowed, Sidekiq is not safe to run on Redis Cluster" if !safe && symbolized_options.key?(:nodes)
|
101
21
|
|
102
|
-
|
22
|
+
size = symbolized_options.delete(:size) || 5
|
23
|
+
pool_timeout = symbolized_options.delete(:pool_timeout) || 1
|
24
|
+
pool_name = symbolized_options.delete(:pool_name)
|
103
25
|
|
104
|
-
|
105
|
-
|
26
|
+
# Default timeout in redis-client is 1 second, which can be too aggressive
|
27
|
+
# if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
|
28
|
+
# it can be easy to get the occasional ReadTimeoutError. You can still provide
|
29
|
+
# a smaller timeout explicitly:
|
30
|
+
# config.redis = { url: "...", timeout: 1 }
|
31
|
+
symbolized_options[:timeout] ||= 3
|
106
32
|
|
107
|
-
redis_config =
|
108
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
33
|
+
redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
|
34
|
+
ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
|
109
35
|
redis_config.new_client
|
110
36
|
end
|
111
37
|
end
|
112
38
|
|
113
39
|
private
|
114
40
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
41
|
+
def deep_symbolize_keys(object)
|
42
|
+
case object
|
43
|
+
when Hash
|
44
|
+
object.each_with_object({}) do |(key, value), result|
|
45
|
+
result[key.to_sym] = deep_symbolize_keys(value)
|
46
|
+
end
|
47
|
+
when Array
|
48
|
+
object.map { |e| deep_symbolize_keys(e) }
|
49
|
+
else
|
50
|
+
object
|
51
|
+
end
|
125
52
|
end
|
126
53
|
|
127
|
-
def
|
54
|
+
def scrub(options)
|
128
55
|
redacted = "REDACTED"
|
129
56
|
|
130
57
|
# Deep clone so we can muck with these options all we want and exclude
|
@@ -136,17 +63,19 @@ module Sidekiq
|
|
136
63
|
uri.password = redacted
|
137
64
|
scrubbed_options[:url] = uri.to_s
|
138
65
|
end
|
139
|
-
if scrubbed_options[:password]
|
140
|
-
|
141
|
-
end
|
66
|
+
scrubbed_options[:password] = redacted if scrubbed_options[:password]
|
67
|
+
scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
|
142
68
|
scrubbed_options[:sentinels]&.each do |sentinel|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
69
|
+
if sentinel.is_a?(String)
|
70
|
+
if (uri = URI(sentinel)) && uri.password
|
71
|
+
uri.password = redacted
|
72
|
+
sentinel.replace(uri.to_s)
|
73
|
+
end
|
74
|
+
elsif sentinel[:password]
|
75
|
+
sentinel[:password] = redacted
|
76
|
+
end
|
149
77
|
end
|
78
|
+
scrubbed_options
|
150
79
|
end
|
151
80
|
|
152
81
|
def determine_redis_provider
|
@@ -168,9 +97,7 @@ module Sidekiq
|
|
168
97
|
EOM
|
169
98
|
end
|
170
99
|
|
171
|
-
ENV[
|
172
|
-
p || "REDIS_URL"
|
173
|
-
]
|
100
|
+
ENV[p.to_s] || ENV["REDIS_URL"]
|
174
101
|
end
|
175
102
|
end
|
176
103
|
end
|