sidekiq 6.5.8 → 7.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +133 -15
- data/README.md +40 -32
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +204 -118
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +114 -128
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +57 -74
- data/lib/sidekiq/client.rb +63 -37
- data/lib/sidekiq/component.rb +4 -1
- data/lib/sidekiq/config.rb +278 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +11 -14
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +17 -14
- data/lib/sidekiq/job_util.rb +49 -15
- data/lib/sidekiq/launcher.rb +66 -62
- data/lib/sidekiq/logger.rb +1 -26
- data/lib/sidekiq/manager.rb +9 -11
- data/lib/sidekiq/metrics/query.rb +4 -4
- data/lib/sidekiq/metrics/shared.rb +7 -6
- data/lib/sidekiq/metrics/tracking.rb +20 -18
- data/lib/sidekiq/middleware/chain.rb +19 -18
- data/lib/sidekiq/middleware/current_attributes.rb +52 -20
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +3 -3
- data/lib/sidekiq/processor.rb +21 -27
- data/lib/sidekiq/rails.rb +12 -7
- data/lib/sidekiq/redis_client_adapter.rb +11 -69
- data/lib/sidekiq/redis_connection.rb +11 -111
- data/lib/sidekiq/scheduled.rb +21 -22
- data/lib/sidekiq/testing.rb +5 -33
- data/lib/sidekiq/transaction_aware_client.rb +4 -5
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/application.rb +21 -6
- data/lib/sidekiq/web/csrf_protection.rb +1 -1
- data/lib/sidekiq/web/helpers.rb +19 -18
- data/lib/sidekiq/web.rb +7 -18
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +76 -274
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +18 -0
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -223
- data/web/assets/javascripts/metrics.js +117 -115
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +23 -298
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +2 -7
- data/web/locales/en.yml +76 -70
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +67 -69
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +59 -69
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +20 -18
- data/web/locales/zh-tw.yml +10 -1
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/busy.erb +39 -28
- data/web/views/dashboard.erb +36 -5
- data/web/views/metrics.erb +33 -20
- data/web/views/metrics_for_job.erb +25 -44
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +10 -14
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +43 -39
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/metrics/deploy.rb +0 -47
- data/lib/sidekiq/worker.rb +0 -370
- data/web/assets/javascripts/graph.js +0 -16
- /data/{LICENSE → LICENSE.txt} +0 -0
@@ -7,27 +7,31 @@ module Sidekiq
|
|
7
7
|
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
8
8
|
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
9
9
|
#
|
10
|
+
# For multiple current attributes, pass an array of current attributes.
|
11
|
+
#
|
10
12
|
# @example
|
11
13
|
#
|
12
14
|
# # in your initializer
|
13
15
|
# require "sidekiq/middleware/current_attributes"
|
14
16
|
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
17
|
+
# # or multiple current attributes
|
18
|
+
# Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
|
15
19
|
#
|
16
20
|
module CurrentAttributes
|
17
21
|
class Save
|
18
22
|
include Sidekiq::ClientMiddleware
|
19
23
|
|
20
|
-
def initialize(
|
21
|
-
@
|
24
|
+
def initialize(cattrs)
|
25
|
+
@cattrs = cattrs
|
22
26
|
end
|
23
27
|
|
24
28
|
def call(_, job, _, _)
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
job
|
29
|
-
|
30
|
-
job[
|
29
|
+
@cattrs.each do |(key, strklass)|
|
30
|
+
if !job.has_key?(key)
|
31
|
+
attrs = strklass.constantize.attributes
|
32
|
+
# Retries can push the job N times, we don't
|
33
|
+
# want retries to reset cattr. #5692, #5090
|
34
|
+
job[key] = attrs if attrs.any?
|
31
35
|
end
|
32
36
|
end
|
33
37
|
yield
|
@@ -37,26 +41,54 @@ module Sidekiq
|
|
37
41
|
class Load
|
38
42
|
include Sidekiq::ServerMiddleware
|
39
43
|
|
40
|
-
def initialize(
|
41
|
-
@
|
44
|
+
def initialize(cattrs)
|
45
|
+
@cattrs = cattrs
|
42
46
|
end
|
43
47
|
|
44
48
|
def call(_, job, _, &block)
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
+
cattrs_to_reset = []
|
50
|
+
|
51
|
+
@cattrs.each do |(key, strklass)|
|
52
|
+
if job.has_key?(key)
|
53
|
+
constklass = strklass.constantize
|
54
|
+
cattrs_to_reset << constklass
|
55
|
+
|
56
|
+
job[key].each do |(attribute, value)|
|
57
|
+
constklass.public_send("#{attribute}=", value)
|
58
|
+
end
|
59
|
+
end
|
49
60
|
end
|
61
|
+
|
62
|
+
yield
|
63
|
+
ensure
|
64
|
+
cattrs_to_reset.each(&:reset)
|
50
65
|
end
|
51
66
|
end
|
52
67
|
|
53
|
-
|
54
|
-
|
55
|
-
|
68
|
+
class << self
|
69
|
+
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
70
|
+
cattrs = build_cattrs_hash(klass_or_array)
|
71
|
+
|
72
|
+
config.client_middleware.add Save, cattrs
|
73
|
+
config.server_middleware.add Load, cattrs
|
56
74
|
end
|
57
|
-
|
58
|
-
|
59
|
-
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def build_cattrs_hash(klass_or_array)
|
79
|
+
if klass_or_array.is_a?(Array)
|
80
|
+
{}.tap do |hash|
|
81
|
+
klass_or_array.each_with_index do |klass, index|
|
82
|
+
hash[key_at(index)] = klass.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
else
|
86
|
+
{key_at(0) => klass_or_array.to_s}
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def key_at(index)
|
91
|
+
(index == 0) ? "cattr" : "cattr_#{index}"
|
60
92
|
end
|
61
93
|
end
|
62
94
|
end
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -16,8 +16,6 @@ class Sidekiq::Monitor
|
|
16
16
|
return
|
17
17
|
end
|
18
18
|
send(section)
|
19
|
-
rescue => e
|
20
|
-
abort "Couldn't get status: #{e}"
|
21
19
|
end
|
22
20
|
|
23
21
|
def all
|
@@ -49,10 +47,25 @@ class Sidekiq::Monitor
|
|
49
47
|
def processes
|
50
48
|
puts "---- Processes (#{process_set.size}) ----"
|
51
49
|
process_set.each_with_index do |process, index|
|
50
|
+
# Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
|
51
|
+
#
|
52
|
+
# Before:
|
53
|
+
# ["default", "critical"]
|
54
|
+
#
|
55
|
+
# After:
|
56
|
+
# {"default" => 1, "critical" => 10}
|
57
|
+
queues =
|
58
|
+
if process["weights"]
|
59
|
+
process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
|
60
|
+
else
|
61
|
+
process["queues"].sort
|
62
|
+
end
|
63
|
+
|
52
64
|
puts "#{process["identity"]} #{tags_for(process)}"
|
53
65
|
puts " Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
|
54
66
|
puts " Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
|
55
|
-
puts " Queues: #{split_multiline(
|
67
|
+
puts " Queues: #{split_multiline(queues, pad: 11)}"
|
68
|
+
puts " Version: #{process["version"] || "Unknown"}" if process["version"] != Sidekiq::VERSION
|
56
69
|
puts "" unless (index + 1) == process_set.size
|
57
70
|
end
|
58
71
|
end
|
@@ -101,7 +114,7 @@ class Sidekiq::Monitor
|
|
101
114
|
tags = [
|
102
115
|
process["tag"],
|
103
116
|
process["labels"],
|
104
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
117
|
+
((process["quiet"] == "true") ? "quiet" : nil)
|
105
118
|
].flatten.compact
|
106
119
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
107
120
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Sidekiq
|
4
4
|
module Paginator
|
5
5
|
def page(key, pageidx = 1, page_size = 25, opts = nil)
|
6
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
6
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
7
7
|
pageidx = current_page - 1
|
8
8
|
total_size = 0
|
9
9
|
items = []
|
@@ -19,7 +19,7 @@ module Sidekiq
|
|
19
19
|
total_size, items = conn.multi { |transaction|
|
20
20
|
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
transaction.
|
22
|
+
transaction.zrange(key, starting, ending, "REV", withscores: true)
|
23
23
|
else
|
24
24
|
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
@@ -45,7 +45,7 @@ module Sidekiq
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def page_items(items, pageidx = 1, page_size = 25)
|
48
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
48
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
49
49
|
pageidx = current_page - 1
|
50
50
|
starting = pageidx * page_size
|
51
51
|
items = items.to_a
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -26,18 +26,18 @@ module Sidekiq
|
|
26
26
|
|
27
27
|
attr_reader :thread
|
28
28
|
attr_reader :job
|
29
|
+
attr_reader :capsule
|
29
30
|
|
30
|
-
def initialize(
|
31
|
+
def initialize(capsule, &block)
|
32
|
+
@config = @capsule = capsule
|
31
33
|
@callback = block
|
32
34
|
@down = false
|
33
35
|
@done = false
|
34
36
|
@job = nil
|
35
37
|
@thread = nil
|
36
|
-
@
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
-
@retrier = Sidekiq::JobRetry.new(options)
|
38
|
+
@reloader = Sidekiq.default_configuration[:reloader]
|
39
|
+
@job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
|
40
|
+
@retrier = Sidekiq::JobRetry.new(capsule)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -59,12 +59,16 @@ module Sidekiq
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def start
|
62
|
-
@thread ||= safe_thread("processor", &method(:run))
|
62
|
+
@thread ||= safe_thread("#{config.name}/processor", &method(:run))
|
63
63
|
end
|
64
64
|
|
65
65
|
private unless $TESTING
|
66
66
|
|
67
67
|
def run
|
68
|
+
# By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
|
69
|
+
# instead of the global pool in +Sidekiq::Config#redis_pool+.
|
70
|
+
Thread.current[:sidekiq_capsule] = @capsule
|
71
|
+
|
68
72
|
process_one until @done
|
69
73
|
@callback.call(self)
|
70
74
|
rescue Sidekiq::Shutdown
|
@@ -80,7 +84,7 @@ module Sidekiq
|
|
80
84
|
end
|
81
85
|
|
82
86
|
def get_one
|
83
|
-
uow =
|
87
|
+
uow = capsule.fetcher.retrieve_work
|
84
88
|
if @down
|
85
89
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
90
|
@down = nil
|
@@ -129,7 +133,7 @@ module Sidekiq
|
|
129
133
|
# the Reloader. It handles code loading, db connection management, etc.
|
130
134
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
135
|
@reloader.call do
|
132
|
-
klass =
|
136
|
+
klass = Object.const_get(job_hash["class"])
|
133
137
|
inst = klass.new
|
134
138
|
inst.jid = job_hash["jid"]
|
135
139
|
@retrier.local(inst, jobstr, queue) do
|
@@ -142,6 +146,9 @@ module Sidekiq
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
149
|
+
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
150
|
+
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
151
|
+
|
145
152
|
def process(uow)
|
146
153
|
jobstr = uow.job
|
147
154
|
queue = uow.queue_name
|
@@ -153,11 +160,11 @@ module Sidekiq
|
|
153
160
|
rescue => ex
|
154
161
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
162
|
now = Time.now.to_f
|
156
|
-
|
163
|
+
redis do |conn|
|
157
164
|
conn.multi do |xa|
|
158
165
|
xa.zadd("dead", now.to_s, jobstr)
|
159
|
-
xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
|
160
|
-
xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
|
166
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
167
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
161
168
|
end
|
162
169
|
end
|
163
170
|
return uow.acknowledge
|
@@ -166,7 +173,7 @@ module Sidekiq
|
|
166
173
|
ack = false
|
167
174
|
begin
|
168
175
|
dispatch(job_hash, queue, jobstr) do |inst|
|
169
|
-
|
176
|
+
config.server_middleware.invoke(inst, job_hash, queue) do
|
170
177
|
execute_job(inst, job_hash["args"])
|
171
178
|
end
|
172
179
|
end
|
@@ -191,7 +198,7 @@ module Sidekiq
|
|
191
198
|
ensure
|
192
199
|
if ack
|
193
200
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
194
|
-
Thread.handle_interrupt(
|
201
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
195
202
|
uow.acknowledge
|
196
203
|
end
|
197
204
|
end
|
@@ -269,18 +276,5 @@ module Sidekiq
|
|
269
276
|
PROCESSED.incr
|
270
277
|
end
|
271
278
|
end
|
272
|
-
|
273
|
-
def constantize(str)
|
274
|
-
return Object.const_get(str) unless str.include?("::")
|
275
|
-
|
276
|
-
names = str.split("::")
|
277
|
-
names.shift if names.empty? || names.first.empty?
|
278
|
-
|
279
|
-
names.inject(Object) do |constant, name|
|
280
|
-
# the false flag limits search for name to under the constant namespace
|
281
|
-
# which mimics Rails' behaviour
|
282
|
-
constant.const_get(name, false)
|
283
|
-
end
|
284
|
-
end
|
285
279
|
end
|
286
280
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq/job"
|
4
|
+
require "rails"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Rails < ::Rails::Engine
|
@@ -10,7 +11,8 @@ module Sidekiq
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def call
|
13
|
-
|
14
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
15
|
+
@app.reloader.wrap(**params) do
|
14
16
|
yield
|
15
17
|
end
|
16
18
|
end
|
@@ -18,11 +20,15 @@ module Sidekiq
|
|
18
20
|
def inspect
|
19
21
|
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
22
|
end
|
23
|
+
|
24
|
+
def to_json(*)
|
25
|
+
Sidekiq.dump_json(inspect)
|
26
|
+
end
|
21
27
|
end
|
22
28
|
|
23
29
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
30
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
-
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
31
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
26
32
|
# manually retried, don't automatically die, etc.
|
27
33
|
#
|
28
34
|
# class SomeJob < ActiveJob::Base
|
@@ -48,11 +54,10 @@ module Sidekiq
|
|
48
54
|
end
|
49
55
|
end
|
50
56
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
"default_worker_options=": :default_job_options=)
|
57
|
+
initializer "sidekiq.backtrace_cleaner" do
|
58
|
+
Sidekiq.configure_server do |config|
|
59
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
60
|
+
end
|
56
61
|
end
|
57
62
|
|
58
63
|
# This hook happens after all initializers are run, just before returning
|
@@ -1,15 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "
|
3
|
+
require "set"
|
4
4
|
require "redis_client"
|
5
5
|
require "redis_client/decorator"
|
6
|
-
require "uri"
|
7
6
|
|
8
7
|
module Sidekiq
|
9
8
|
class RedisClientAdapter
|
10
9
|
BaseError = RedisClient::Error
|
11
10
|
CommandError = RedisClient::CommandError
|
12
11
|
|
12
|
+
# You can add/remove items or clear the whole thing if you don't want deprecation warnings.
|
13
|
+
DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
|
14
|
+
|
13
15
|
module CompatMethods
|
14
16
|
def info
|
15
17
|
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
@@ -19,30 +21,12 @@ module Sidekiq
|
|
19
21
|
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
22
|
end
|
21
23
|
|
22
|
-
def brpoplpush(*args)
|
23
|
-
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
-
end
|
25
|
-
|
26
|
-
def brpop(*args)
|
27
|
-
@client.blocking_call(false, "BRPOP", *args)
|
28
|
-
end
|
29
|
-
|
30
|
-
def set(*args)
|
31
|
-
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
-
end
|
33
|
-
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
-
|
35
|
-
def sismember(*args)
|
36
|
-
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
-
end
|
38
|
-
|
39
|
-
def exists?(key)
|
40
|
-
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
-
end
|
42
|
-
|
43
24
|
private
|
44
25
|
|
26
|
+
# this allows us to use methods like `conn.hmset(...)` instead of having to use
|
27
|
+
# redis-client's native `conn.call("hmset", ...)`
|
45
28
|
def method_missing(*args, &block)
|
29
|
+
warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
|
46
30
|
@client.call(*args, *block)
|
47
31
|
end
|
48
32
|
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
@@ -55,47 +39,8 @@ module Sidekiq
|
|
55
39
|
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
40
|
|
57
41
|
class CompatClient
|
58
|
-
|
59
|
-
|
60
|
-
undef_method method
|
61
|
-
end
|
62
|
-
|
63
|
-
def disconnect!
|
64
|
-
@client.close
|
65
|
-
end
|
66
|
-
|
67
|
-
def connection
|
68
|
-
{id: @client.id}
|
69
|
-
end
|
70
|
-
|
71
|
-
def redis
|
72
|
-
self
|
73
|
-
end
|
74
|
-
|
75
|
-
def _client
|
76
|
-
@client
|
77
|
-
end
|
78
|
-
|
79
|
-
def message
|
80
|
-
yield nil, @queue.pop
|
81
|
-
end
|
82
|
-
|
83
|
-
# NB: this method does not return
|
84
|
-
def subscribe(chan)
|
85
|
-
@queue = ::Queue.new
|
86
|
-
|
87
|
-
pubsub = @client.pubsub
|
88
|
-
pubsub.call("subscribe", chan)
|
89
|
-
|
90
|
-
loop do
|
91
|
-
evt = pubsub.next_event
|
92
|
-
next if evt.nil?
|
93
|
-
next unless evt[0] == "message" && evt[1] == chan
|
94
|
-
|
95
|
-
(_, _, msg) = evt
|
96
|
-
@queue << msg
|
97
|
-
yield self
|
98
|
-
end
|
42
|
+
def config
|
43
|
+
@client.config
|
99
44
|
end
|
100
45
|
end
|
101
46
|
|
@@ -118,9 +63,8 @@ module Sidekiq
|
|
118
63
|
opts = options.dup
|
119
64
|
|
120
65
|
if opts[:namespace]
|
121
|
-
|
122
|
-
"Either use the redis adapter or remove the namespace."
|
123
|
-
Kernel.exit(-127)
|
66
|
+
raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
67
|
+
"Either use the redis adapter or remove the namespace."
|
124
68
|
end
|
125
69
|
|
126
70
|
opts.delete(:size)
|
@@ -150,5 +94,3 @@ module Sidekiq
|
|
150
94
|
end
|
151
95
|
end
|
152
96
|
end
|
153
|
-
|
154
|
-
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -1,128 +1,32 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "connection_pool"
|
4
|
-
require "redis"
|
5
4
|
require "uri"
|
5
|
+
require "sidekiq/redis_client_adapter"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
module RedisConnection
|
9
|
-
class RedisAdapter
|
10
|
-
BaseError = Redis::BaseError
|
11
|
-
CommandError = Redis::CommandError
|
12
|
-
|
13
|
-
def initialize(options)
|
14
|
-
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
-
@options = options
|
16
|
-
end
|
17
|
-
|
18
|
-
def new_client
|
19
|
-
namespace = @options[:namespace]
|
20
|
-
|
21
|
-
client = Redis.new client_opts(@options)
|
22
|
-
if namespace
|
23
|
-
begin
|
24
|
-
require "redis/namespace"
|
25
|
-
Redis::Namespace.new(namespace, redis: client)
|
26
|
-
rescue LoadError
|
27
|
-
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
28
|
-
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
29
|
-
exit(-127)
|
30
|
-
end
|
31
|
-
else
|
32
|
-
client
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def client_opts(options)
|
39
|
-
opts = options.dup
|
40
|
-
if opts[:namespace]
|
41
|
-
opts.delete(:namespace)
|
42
|
-
end
|
43
|
-
|
44
|
-
if opts[:network_timeout]
|
45
|
-
opts[:timeout] = opts[:network_timeout]
|
46
|
-
opts.delete(:network_timeout)
|
47
|
-
end
|
48
|
-
|
49
|
-
# Issue #3303, redis-rb will silently retry an operation.
|
50
|
-
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
51
|
-
# is performed twice but I believe this is much, much rarer
|
52
|
-
# than the reconnect silently fixing a problem; we keep it
|
53
|
-
# on by default.
|
54
|
-
opts[:reconnect_attempts] ||= 1
|
55
|
-
|
56
|
-
opts
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
@adapter = RedisAdapter
|
61
|
-
|
62
9
|
class << self
|
63
|
-
attr_reader :adapter
|
64
|
-
|
65
|
-
# RedisConnection.adapter = :redis
|
66
|
-
# RedisConnection.adapter = :redis_client
|
67
|
-
def adapter=(adapter)
|
68
|
-
raise "no" if adapter == self
|
69
|
-
result = case adapter
|
70
|
-
when :redis
|
71
|
-
RedisAdapter
|
72
|
-
when Class
|
73
|
-
adapter
|
74
|
-
else
|
75
|
-
require "sidekiq/#{adapter}_adapter"
|
76
|
-
nil
|
77
|
-
end
|
78
|
-
@adapter = result if result
|
79
|
-
end
|
80
|
-
|
81
10
|
def create(options = {})
|
82
11
|
symbolized_options = options.transform_keys(&:to_sym)
|
12
|
+
symbolized_options[:url] ||= determine_redis_provider
|
83
13
|
|
84
|
-
|
85
|
-
|
86
|
-
end
|
87
|
-
|
88
|
-
size = if symbolized_options[:size]
|
89
|
-
symbolized_options[:size]
|
90
|
-
elsif Sidekiq.server?
|
91
|
-
# Give ourselves plenty of connections. pool is lazy
|
92
|
-
# so we won't create them until we need them.
|
93
|
-
Sidekiq[:concurrency] + 5
|
94
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
95
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
96
|
-
else
|
97
|
-
5
|
98
|
-
end
|
99
|
-
|
100
|
-
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
14
|
+
logger = symbolized_options.delete(:logger)
|
15
|
+
logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
|
101
16
|
|
102
|
-
|
103
|
-
|
17
|
+
size = symbolized_options.delete(:size) || 5
|
18
|
+
pool_timeout = symbolized_options.delete(:pool_timeout) || 1
|
19
|
+
pool_name = symbolized_options.delete(:pool_name)
|
104
20
|
|
105
|
-
redis_config =
|
106
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
21
|
+
redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
|
22
|
+
ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
|
107
23
|
redis_config.new_client
|
108
24
|
end
|
109
25
|
end
|
110
26
|
|
111
27
|
private
|
112
28
|
|
113
|
-
|
114
|
-
#
|
115
|
-
# We need a connection for each Processor.
|
116
|
-
# We need a connection for Pro's real-time change listener
|
117
|
-
# We need a connection to various features to call Redis every few seconds:
|
118
|
-
# - the process heartbeat.
|
119
|
-
# - enterprise's leader election
|
120
|
-
# - enterprise's cron support
|
121
|
-
def verify_sizing(size, concurrency)
|
122
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
123
|
-
end
|
124
|
-
|
125
|
-
def log_info(options)
|
29
|
+
def scrub(options)
|
126
30
|
redacted = "REDACTED"
|
127
31
|
|
128
32
|
# Deep clone so we can muck with these options all we want and exclude
|
@@ -140,11 +44,7 @@ module Sidekiq
|
|
140
44
|
scrubbed_options[:sentinels]&.each do |sentinel|
|
141
45
|
sentinel[:password] = redacted if sentinel[:password]
|
142
46
|
end
|
143
|
-
|
144
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
145
|
-
else
|
146
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
147
|
-
end
|
47
|
+
scrubbed_options
|
148
48
|
end
|
149
49
|
|
150
50
|
def determine_redis_provider
|