sidekiq 6.4.1 → 7.0.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +172 -12
- data/README.md +41 -33
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +188 -114
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +275 -161
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +83 -88
- data/lib/sidekiq/client.rb +55 -43
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +270 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +21 -22
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +76 -54
- data/lib/sidekiq/job_util.rb +59 -19
- data/lib/sidekiq/launcher.rb +90 -82
- data/lib/sidekiq/logger.rb +6 -45
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +16 -17
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +11 -3
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +12 -10
- data/lib/sidekiq/redis_client_adapter.rb +115 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +65 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +40 -9
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +34 -20
- data/lib/sidekiq/web.rb +7 -14
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -207
- data/sidekiq.gemspec +20 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +66 -299
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +42 -26
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +71 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +63 -28
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -362
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
29
|
+
attr_reader :capsule
|
30
30
|
|
31
|
-
def initialize(
|
32
|
-
@
|
31
|
+
def initialize(capsule, &block)
|
32
|
+
@config = @capsule = capsule
|
33
|
+
@callback = block
|
33
34
|
@down = false
|
34
35
|
@done = false
|
35
36
|
@job = nil
|
36
37
|
@thread = nil
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@
|
40
|
-
@retrier = Sidekiq::JobRetry.new
|
38
|
+
@reloader = Sidekiq.default_configuration[:reloader]
|
39
|
+
@job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
|
40
|
+
@retrier = Sidekiq::JobRetry.new(capsule)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -59,33 +59,37 @@ module Sidekiq
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def start
|
62
|
-
@thread ||= safe_thread("processor", &method(:run))
|
62
|
+
@thread ||= safe_thread("#{config.name}/processor", &method(:run))
|
63
63
|
end
|
64
64
|
|
65
65
|
private unless $TESTING
|
66
66
|
|
67
67
|
def run
|
68
|
+
# By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
|
69
|
+
# instead of the global pool in +Sidekiq::Config#redis_pool+.
|
70
|
+
Thread.current[:sidekiq_capsule] = @capsule
|
71
|
+
|
68
72
|
process_one until @done
|
69
|
-
@
|
73
|
+
@callback.call(self)
|
70
74
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
75
|
+
@callback.call(self)
|
72
76
|
rescue Exception => ex
|
73
|
-
@
|
77
|
+
@callback.call(self, ex)
|
74
78
|
end
|
75
79
|
|
76
|
-
def process_one
|
80
|
+
def process_one(&block)
|
77
81
|
@job = fetch
|
78
82
|
process(@job) if @job
|
79
83
|
@job = nil
|
80
84
|
end
|
81
85
|
|
82
86
|
def get_one
|
83
|
-
|
87
|
+
uow = capsule.fetcher.retrieve_work
|
84
88
|
if @down
|
85
89
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
90
|
@down = nil
|
87
91
|
end
|
88
|
-
|
92
|
+
uow
|
89
93
|
rescue Sidekiq::Shutdown
|
90
94
|
rescue => ex
|
91
95
|
handle_fetch_exception(ex)
|
@@ -129,11 +133,11 @@ module Sidekiq
|
|
129
133
|
# the Reloader. It handles code loading, db connection management, etc.
|
130
134
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
135
|
@reloader.call do
|
132
|
-
klass =
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
136
|
+
klass = Object.const_get(job_hash["class"])
|
137
|
+
inst = klass.new
|
138
|
+
inst.jid = job_hash["jid"]
|
139
|
+
@retrier.local(inst, jobstr, queue) do
|
140
|
+
yield inst
|
137
141
|
end
|
138
142
|
end
|
139
143
|
end
|
@@ -142,9 +146,12 @@ module Sidekiq
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
145
|
-
|
146
|
-
|
147
|
-
|
149
|
+
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
150
|
+
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
151
|
+
|
152
|
+
def process(uow)
|
153
|
+
jobstr = uow.job
|
154
|
+
queue = uow.queue_name
|
148
155
|
|
149
156
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
157
|
job_hash = nil
|
@@ -152,16 +159,22 @@ module Sidekiq
|
|
152
159
|
job_hash = Sidekiq.load_json(jobstr)
|
153
160
|
rescue => ex
|
154
161
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
-
|
156
|
-
|
157
|
-
|
162
|
+
now = Time.now.to_f
|
163
|
+
redis do |conn|
|
164
|
+
conn.multi do |xa|
|
165
|
+
xa.zadd("dead", now.to_s, jobstr)
|
166
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
167
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
168
|
+
end
|
169
|
+
end
|
170
|
+
return uow.acknowledge
|
158
171
|
end
|
159
172
|
|
160
173
|
ack = false
|
161
174
|
begin
|
162
|
-
dispatch(job_hash, queue, jobstr) do |
|
163
|
-
|
164
|
-
execute_job(
|
175
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
176
|
+
config.server_middleware.invoke(inst, job_hash, queue) do
|
177
|
+
execute_job(inst, job_hash["args"])
|
165
178
|
end
|
166
179
|
end
|
167
180
|
ack = true
|
@@ -174,7 +187,7 @@ module Sidekiq
|
|
174
187
|
# signals that we created a retry successfully. We can acknowlege the job.
|
175
188
|
ack = true
|
176
189
|
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash
|
190
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
178
191
|
raise e
|
179
192
|
rescue Exception => ex
|
180
193
|
# Unexpected error! This is very bad and indicates an exception that got past
|
@@ -185,15 +198,15 @@ module Sidekiq
|
|
185
198
|
ensure
|
186
199
|
if ack
|
187
200
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
|
-
Thread.handle_interrupt(
|
189
|
-
|
201
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
202
|
+
uow.acknowledge
|
190
203
|
end
|
191
204
|
end
|
192
205
|
end
|
193
206
|
end
|
194
207
|
|
195
|
-
def execute_job(
|
196
|
-
|
208
|
+
def execute_job(inst, cloned_args)
|
209
|
+
inst.perform(*cloned_args)
|
197
210
|
end
|
198
211
|
|
199
212
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +232,39 @@ module Sidekiq
|
|
219
232
|
end
|
220
233
|
|
221
234
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
235
|
+
class SharedWorkState
|
223
236
|
def initialize
|
224
|
-
@
|
237
|
+
@work_state = {}
|
225
238
|
@lock = Mutex.new
|
226
239
|
end
|
227
240
|
|
228
241
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
242
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
243
|
end
|
231
244
|
|
232
245
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
246
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
247
|
end
|
235
248
|
|
236
249
|
def dup
|
237
|
-
@lock.synchronize { @
|
250
|
+
@lock.synchronize { @work_state.dup }
|
238
251
|
end
|
239
252
|
|
240
253
|
def size
|
241
|
-
@lock.synchronize { @
|
254
|
+
@lock.synchronize { @work_state.size }
|
242
255
|
end
|
243
256
|
|
244
257
|
def clear
|
245
|
-
@lock.synchronize { @
|
258
|
+
@lock.synchronize { @work_state.clear }
|
246
259
|
end
|
247
260
|
end
|
248
261
|
|
249
262
|
PROCESSED = Counter.new
|
250
263
|
FAILURE = Counter.new
|
251
|
-
|
264
|
+
WORK_STATE = SharedWorkState.new
|
252
265
|
|
253
266
|
def stats(jobstr, queue)
|
254
|
-
|
267
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
268
|
|
256
269
|
begin
|
257
270
|
yield
|
@@ -259,22 +272,9 @@ module Sidekiq
|
|
259
272
|
FAILURE.incr
|
260
273
|
raise
|
261
274
|
ensure
|
262
|
-
|
275
|
+
WORK_STATE.delete(tid)
|
263
276
|
PROCESSED.incr
|
264
277
|
end
|
265
278
|
end
|
266
|
-
|
267
|
-
def constantize(str)
|
268
|
-
return Object.const_get(str) unless str.include?("::")
|
269
|
-
|
270
|
-
names = str.split("::")
|
271
|
-
names.shift if names.empty? || names.first.empty?
|
272
|
-
|
273
|
-
names.inject(Object) do |constant, name|
|
274
|
-
# the false flag limits search for name to under the constant namespace
|
275
|
-
# which mimics Rails' behaviour
|
276
|
-
constant.const_get(name, false)
|
277
|
-
end
|
278
|
-
end
|
279
279
|
end
|
280
280
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
|
+
require "rails"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Rails < ::Rails::Engine
|
@@ -10,7 +11,8 @@ module Sidekiq
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def call
|
13
|
-
|
14
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
15
|
+
@app.reloader.wrap(**params) do
|
14
16
|
yield
|
15
17
|
end
|
16
18
|
end
|
@@ -22,7 +24,7 @@ module Sidekiq
|
|
22
24
|
|
23
25
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
26
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
-
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
27
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
26
28
|
# manually retried, don't automatically die, etc.
|
27
29
|
#
|
28
30
|
# class SomeJob < ActiveJob::Base
|
@@ -33,17 +35,17 @@ module Sidekiq
|
|
33
35
|
# end
|
34
36
|
initializer "sidekiq.active_job_integration" do
|
35
37
|
ActiveSupport.on_load(:active_job) do
|
36
|
-
include ::Sidekiq::
|
38
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
39
|
end
|
38
40
|
end
|
39
41
|
|
40
42
|
initializer "sidekiq.rails_logger" do
|
41
|
-
Sidekiq.configure_server do |
|
42
|
-
# This is the integration code necessary so that if
|
43
|
+
Sidekiq.configure_server do |config|
|
44
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
45
|
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
46
|
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
-
unless ::Rails.logger ==
|
46
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(
|
47
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
48
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
49
|
end
|
48
50
|
end
|
49
51
|
end
|
@@ -53,8 +55,8 @@ module Sidekiq
|
|
53
55
|
#
|
54
56
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
55
57
|
config.after_initialize do
|
56
|
-
Sidekiq.configure_server do |
|
57
|
-
|
58
|
+
Sidekiq.configure_server do |config|
|
59
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
58
60
|
end
|
59
61
|
end
|
60
62
|
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "redis_client"
|
4
|
+
require "redis_client/decorator"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class RedisClientAdapter
|
8
|
+
BaseError = RedisClient::Error
|
9
|
+
CommandError = RedisClient::CommandError
|
10
|
+
|
11
|
+
module CompatMethods
|
12
|
+
# TODO Deprecate and remove this
|
13
|
+
def info
|
14
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
15
|
+
end
|
16
|
+
|
17
|
+
# TODO Deprecate and remove this
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
# this allows us to use methods like `conn.hmset(...)` instead of having to use
|
25
|
+
# redis-client's native `conn.call("hmset", ...)`
|
26
|
+
def method_missing(*args, &block)
|
27
|
+
@client.call(*args, *block)
|
28
|
+
end
|
29
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
30
|
+
|
31
|
+
def respond_to_missing?(name, include_private = false)
|
32
|
+
super # Appease the linter. We can't tell what is a valid command.
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
37
|
+
|
38
|
+
class CompatClient
|
39
|
+
def config
|
40
|
+
@client.config
|
41
|
+
end
|
42
|
+
|
43
|
+
def message
|
44
|
+
yield nil, @queue.pop
|
45
|
+
end
|
46
|
+
|
47
|
+
# NB: this method does not return
|
48
|
+
def subscribe(chan)
|
49
|
+
@queue = ::Queue.new
|
50
|
+
|
51
|
+
pubsub = @client.pubsub
|
52
|
+
pubsub.call("subscribe", chan)
|
53
|
+
|
54
|
+
loop do
|
55
|
+
evt = pubsub.next_event
|
56
|
+
next if evt.nil?
|
57
|
+
next unless evt[0] == "message" && evt[1] == chan
|
58
|
+
|
59
|
+
(_, _, msg) = evt
|
60
|
+
@queue << msg
|
61
|
+
yield self
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def initialize(options)
|
67
|
+
opts = client_opts(options)
|
68
|
+
@config = if opts.key?(:sentinels)
|
69
|
+
RedisClient.sentinel(**opts)
|
70
|
+
else
|
71
|
+
RedisClient.config(**opts)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def new_client
|
76
|
+
CompatClient.new(@config.new_client)
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
|
81
|
+
def client_opts(options)
|
82
|
+
opts = options.dup
|
83
|
+
|
84
|
+
if opts[:namespace]
|
85
|
+
raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
86
|
+
"Either use the redis adapter or remove the namespace."
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.delete(:size)
|
90
|
+
opts.delete(:pool_timeout)
|
91
|
+
|
92
|
+
if opts[:network_timeout]
|
93
|
+
opts[:timeout] = opts[:network_timeout]
|
94
|
+
opts.delete(:network_timeout)
|
95
|
+
end
|
96
|
+
|
97
|
+
if opts[:driver]
|
98
|
+
opts[:driver] = opts[:driver].to_sym
|
99
|
+
end
|
100
|
+
|
101
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
102
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
103
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
104
|
+
|
105
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
106
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
107
|
+
# is performed twice but I believe this is much, much rarer
|
108
|
+
# than the reconnect silently fixing a problem; we keep it
|
109
|
+
# on by default.
|
110
|
+
opts[:reconnect_attempts] ||= 1
|
111
|
+
|
112
|
+
opts
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -1,97 +1,32 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "connection_pool"
|
4
|
-
require "redis"
|
5
4
|
require "uri"
|
5
|
+
require "sidekiq/redis_client_adapter"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
8
|
+
module RedisConnection
|
9
9
|
class << self
|
10
10
|
def create(options = {})
|
11
11
|
symbolized_options = options.transform_keys(&:to_sym)
|
12
|
+
symbolized_options[:url] ||= determine_redis_provider
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
size = if symbolized_options[:size]
|
18
|
-
symbolized_options[:size]
|
19
|
-
elsif Sidekiq.server?
|
20
|
-
# Give ourselves plenty of connections. pool is lazy
|
21
|
-
# so we won't create them until we need them.
|
22
|
-
Sidekiq.options[:concurrency] + 5
|
23
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
24
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
25
|
-
else
|
26
|
-
5
|
27
|
-
end
|
14
|
+
logger = symbolized_options.delete(:logger)
|
15
|
+
logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
|
28
16
|
|
29
|
-
|
17
|
+
size = symbolized_options.delete(:size) || 5
|
18
|
+
pool_timeout = symbolized_options.delete(:pool_timeout) || 1
|
19
|
+
pool_name = symbolized_options.delete(:pool_name)
|
30
20
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
build_client(symbolized_options)
|
21
|
+
redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
|
22
|
+
ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
|
23
|
+
redis_config.new_client
|
36
24
|
end
|
37
25
|
end
|
38
26
|
|
39
27
|
private
|
40
28
|
|
41
|
-
|
42
|
-
#
|
43
|
-
# We need a connection for each Processor.
|
44
|
-
# We need a connection for Pro's real-time change listener
|
45
|
-
# We need a connection to various features to call Redis every few seconds:
|
46
|
-
# - the process heartbeat.
|
47
|
-
# - enterprise's leader election
|
48
|
-
# - enterprise's cron support
|
49
|
-
def verify_sizing(size, concurrency)
|
50
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
57
|
-
if namespace
|
58
|
-
begin
|
59
|
-
require "redis/namespace"
|
60
|
-
Redis::Namespace.new(namespace, redis: client)
|
61
|
-
rescue LoadError
|
62
|
-
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
63
|
-
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
64
|
-
exit(-127)
|
65
|
-
end
|
66
|
-
else
|
67
|
-
client
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def client_opts(options)
|
72
|
-
opts = options.dup
|
73
|
-
if opts[:namespace]
|
74
|
-
opts.delete(:namespace)
|
75
|
-
end
|
76
|
-
|
77
|
-
if opts[:network_timeout]
|
78
|
-
opts[:timeout] = opts[:network_timeout]
|
79
|
-
opts.delete(:network_timeout)
|
80
|
-
end
|
81
|
-
|
82
|
-
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
83
|
-
|
84
|
-
# Issue #3303, redis-rb will silently retry an operation.
|
85
|
-
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
86
|
-
# is performed twice but I believe this is much, much rarer
|
87
|
-
# than the reconnect silently fixing a problem; we keep it
|
88
|
-
# on by default.
|
89
|
-
opts[:reconnect_attempts] ||= 1
|
90
|
-
|
91
|
-
opts
|
92
|
-
end
|
93
|
-
|
94
|
-
def log_info(options)
|
29
|
+
def scrub(options)
|
95
30
|
redacted = "REDACTED"
|
96
31
|
|
97
32
|
# Deep clone so we can muck with these options all we want and exclude
|
@@ -109,11 +44,7 @@ module Sidekiq
|
|
109
44
|
scrubbed_options[:sentinels]&.each do |sentinel|
|
110
45
|
sentinel[:password] = redacted if sentinel[:password]
|
111
46
|
end
|
112
|
-
|
113
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
114
|
-
else
|
115
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with redis options #{scrubbed_options}")
|
116
|
-
end
|
47
|
+
scrubbed_options
|
117
48
|
end
|
118
49
|
|
119
50
|
def determine_redis_provider
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|