sidekiq 6.4.1 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +87 -1
- data/README.md +1 -1
- data/bin/sidekiqload +18 -12
- data/lib/sidekiq/api.rb +203 -59
- data/lib/sidekiq/cli.rb +39 -37
- data/lib/sidekiq/client.rb +26 -27
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +1 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
- data/lib/sidekiq/fetch.rb +18 -16
- data/lib/sidekiq/job_retry.rb +73 -52
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +37 -33
- data/lib/sidekiq/logger.rb +5 -19
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +10 -4
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +2 -2
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -49
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +53 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +18 -5
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +28 -5
- data/lib/sidekiq/web.rb +5 -1
- data/lib/sidekiq/worker.rb +22 -14
- data/lib/sidekiq.rb +106 -31
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application.css +45 -3
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/ja.yml +7 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +4 -4
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +27 -8
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
@@ -10,16 +10,18 @@ module Sidekiq::Middleware::I18n
|
|
10
10
|
# Get the current locale and store it in the message
|
11
11
|
# to be sent to Sidekiq.
|
12
12
|
class Client
|
13
|
-
|
14
|
-
|
13
|
+
include Sidekiq::ClientMiddleware
|
14
|
+
def call(_jobclass, job, _queue, _redis)
|
15
|
+
job["locale"] ||= I18n.locale
|
15
16
|
yield
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
19
20
|
# Pull the msg locale out and set the current thread to use it.
|
20
21
|
class Server
|
21
|
-
|
22
|
-
|
22
|
+
include Sidekiq::ServerMiddleware
|
23
|
+
def call(_jobclass, job, _queue, &block)
|
24
|
+
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
# Server-side middleware must import this Module in order
|
3
|
+
# to get access to server resources during `call`.
|
4
|
+
module ServerMiddleware
|
5
|
+
attr_accessor :config
|
6
|
+
def redis_pool
|
7
|
+
config.redis_pool
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
config.logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def redis(&block)
|
15
|
+
config.redis(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# no difference for now
|
20
|
+
ClientMiddleware = ServerMiddleware
|
21
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
data/lib/sidekiq/paginator.rb
CHANGED
@@ -19,9 +19,9 @@ module Sidekiq
|
|
19
19
|
total_size, items = conn.multi { |transaction|
|
20
20
|
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
transaction.zrevrange(key, starting, ending,
|
22
|
+
transaction.zrevrange(key, starting, ending, withscores: true)
|
23
23
|
else
|
24
|
-
transaction.zrange(key, starting, ending,
|
24
|
+
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
30
29
|
|
31
|
-
def initialize(
|
32
|
-
@
|
30
|
+
def initialize(options, &block)
|
31
|
+
@callback = block
|
33
32
|
@down = false
|
34
33
|
@done = false
|
35
34
|
@job = nil
|
36
35
|
@thread = nil
|
36
|
+
@config = options
|
37
37
|
@strategy = options[:fetch]
|
38
38
|
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
39
|
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
-
@retrier = Sidekiq::JobRetry.new
|
40
|
+
@retrier = Sidekiq::JobRetry.new(options)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -66,26 +66,26 @@ module Sidekiq
|
|
66
66
|
|
67
67
|
def run
|
68
68
|
process_one until @done
|
69
|
-
@
|
69
|
+
@callback.call(self)
|
70
70
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
71
|
+
@callback.call(self)
|
72
72
|
rescue Exception => ex
|
73
|
-
@
|
73
|
+
@callback.call(self, ex)
|
74
74
|
end
|
75
75
|
|
76
|
-
def process_one
|
76
|
+
def process_one(&block)
|
77
77
|
@job = fetch
|
78
78
|
process(@job) if @job
|
79
79
|
@job = nil
|
80
80
|
end
|
81
81
|
|
82
82
|
def get_one
|
83
|
-
|
83
|
+
uow = @strategy.retrieve_work
|
84
84
|
if @down
|
85
85
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
86
|
@down = nil
|
87
87
|
end
|
88
|
-
|
88
|
+
uow
|
89
89
|
rescue Sidekiq::Shutdown
|
90
90
|
rescue => ex
|
91
91
|
handle_fetch_exception(ex)
|
@@ -130,10 +130,10 @@ module Sidekiq
|
|
130
130
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
131
|
@reloader.call do
|
132
132
|
klass = constantize(job_hash["class"])
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
133
|
+
inst = klass.new
|
134
|
+
inst.jid = job_hash["jid"]
|
135
|
+
@retrier.local(inst, jobstr, queue) do
|
136
|
+
yield inst
|
137
137
|
end
|
138
138
|
end
|
139
139
|
end
|
@@ -142,9 +142,9 @@ module Sidekiq
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
def process(
|
146
|
-
jobstr =
|
147
|
-
queue =
|
145
|
+
def process(uow)
|
146
|
+
jobstr = uow.job
|
147
|
+
queue = uow.queue_name
|
148
148
|
|
149
149
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
150
|
job_hash = nil
|
@@ -152,16 +152,22 @@ module Sidekiq
|
|
152
152
|
job_hash = Sidekiq.load_json(jobstr)
|
153
153
|
rescue => ex
|
154
154
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
-
|
156
|
-
|
157
|
-
|
155
|
+
now = Time.now.to_f
|
156
|
+
config.redis do |conn|
|
157
|
+
conn.multi do |xa|
|
158
|
+
xa.zadd("dead", now.to_s, jobstr)
|
159
|
+
xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
|
160
|
+
xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
|
161
|
+
end
|
162
|
+
end
|
163
|
+
return uow.acknowledge
|
158
164
|
end
|
159
165
|
|
160
166
|
ack = false
|
161
167
|
begin
|
162
|
-
dispatch(job_hash, queue, jobstr) do |
|
163
|
-
|
164
|
-
execute_job(
|
168
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
169
|
+
@config.server_middleware.invoke(inst, job_hash, queue) do
|
170
|
+
execute_job(inst, job_hash["args"])
|
165
171
|
end
|
166
172
|
end
|
167
173
|
ack = true
|
@@ -174,7 +180,7 @@ module Sidekiq
|
|
174
180
|
# signals that we created a retry successfully. We can acknowlege the job.
|
175
181
|
ack = true
|
176
182
|
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash
|
183
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
178
184
|
raise e
|
179
185
|
rescue Exception => ex
|
180
186
|
# Unexpected error! This is very bad and indicates an exception that got past
|
@@ -186,14 +192,14 @@ module Sidekiq
|
|
186
192
|
if ack
|
187
193
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
194
|
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
|
195
|
+
uow.acknowledge
|
190
196
|
end
|
191
197
|
end
|
192
198
|
end
|
193
199
|
end
|
194
200
|
|
195
|
-
def execute_job(
|
196
|
-
|
201
|
+
def execute_job(inst, cloned_args)
|
202
|
+
inst.perform(*cloned_args)
|
197
203
|
end
|
198
204
|
|
199
205
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +225,39 @@ module Sidekiq
|
|
219
225
|
end
|
220
226
|
|
221
227
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
228
|
+
class SharedWorkState
|
223
229
|
def initialize
|
224
|
-
@
|
230
|
+
@work_state = {}
|
225
231
|
@lock = Mutex.new
|
226
232
|
end
|
227
233
|
|
228
234
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
235
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
236
|
end
|
231
237
|
|
232
238
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
239
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
240
|
end
|
235
241
|
|
236
242
|
def dup
|
237
|
-
@lock.synchronize { @
|
243
|
+
@lock.synchronize { @work_state.dup }
|
238
244
|
end
|
239
245
|
|
240
246
|
def size
|
241
|
-
@lock.synchronize { @
|
247
|
+
@lock.synchronize { @work_state.size }
|
242
248
|
end
|
243
249
|
|
244
250
|
def clear
|
245
|
-
@lock.synchronize { @
|
251
|
+
@lock.synchronize { @work_state.clear }
|
246
252
|
end
|
247
253
|
end
|
248
254
|
|
249
255
|
PROCESSED = Counter.new
|
250
256
|
FAILURE = Counter.new
|
251
|
-
|
257
|
+
WORK_STATE = SharedWorkState.new
|
252
258
|
|
253
259
|
def stats(jobstr, queue)
|
254
|
-
|
260
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
261
|
|
256
262
|
begin
|
257
263
|
yield
|
@@ -259,7 +265,7 @@ module Sidekiq
|
|
259
265
|
FAILURE.incr
|
260
266
|
raise
|
261
267
|
ensure
|
262
|
-
|
268
|
+
WORK_STATE.delete(tid)
|
263
269
|
PROCESSED.incr
|
264
270
|
end
|
265
271
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
@@ -33,28 +33,35 @@ module Sidekiq
|
|
33
33
|
# end
|
34
34
|
initializer "sidekiq.active_job_integration" do
|
35
35
|
ActiveSupport.on_load(:active_job) do
|
36
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
40
|
initializer "sidekiq.rails_logger" do
|
41
|
-
Sidekiq.configure_server do |
|
42
|
-
# This is the integration code necessary so that if
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
43
|
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
44
|
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
-
unless ::Rails.logger ==
|
46
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
47
|
end
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
51
58
|
# This hook happens after all initializers are run, just before returning
|
52
59
|
# from config/environment.rb back to sidekiq/cli.rb.
|
53
60
|
#
|
54
61
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
55
62
|
config.after_initialize do
|
56
|
-
Sidekiq.configure_server do |
|
57
|
-
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
58
65
|
end
|
59
66
|
end
|
60
67
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,8 +5,79 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
16
|
+
end
|
17
|
+
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
20
|
+
|
21
|
+
client = Redis.new client_opts(@options)
|
22
|
+
if namespace
|
23
|
+
begin
|
24
|
+
require "redis/namespace"
|
25
|
+
Redis::Namespace.new(namespace, redis: client)
|
26
|
+
rescue LoadError
|
27
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
28
|
+
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
29
|
+
exit(-127)
|
30
|
+
end
|
31
|
+
else
|
32
|
+
client
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def client_opts(options)
|
39
|
+
opts = options.dup
|
40
|
+
if opts[:namespace]
|
41
|
+
opts.delete(:namespace)
|
42
|
+
end
|
43
|
+
|
44
|
+
if opts[:network_timeout]
|
45
|
+
opts[:timeout] = opts[:network_timeout]
|
46
|
+
opts.delete(:network_timeout)
|
47
|
+
end
|
48
|
+
|
49
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
50
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
51
|
+
# is performed twice but I believe this is much, much rarer
|
52
|
+
# than the reconnect silently fixing a problem; we keep it
|
53
|
+
# on by default.
|
54
|
+
opts[:reconnect_attempts] ||= 1
|
55
|
+
|
56
|
+
opts
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
@adapter = RedisAdapter
|
61
|
+
|
9
62
|
class << self
|
63
|
+
attr_reader :adapter
|
64
|
+
|
65
|
+
# RedisConnection.adapter = :redis
|
66
|
+
# RedisConnection.adapter = :redis_client
|
67
|
+
def adapter=(adapter)
|
68
|
+
raise "no" if adapter == self
|
69
|
+
result = case adapter
|
70
|
+
when :redis
|
71
|
+
RedisAdapter
|
72
|
+
when Class
|
73
|
+
adapter
|
74
|
+
else
|
75
|
+
require "sidekiq/#{adapter}_adapter"
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
@adapter = result if result
|
79
|
+
end
|
80
|
+
|
10
81
|
def create(options = {})
|
11
82
|
symbolized_options = options.transform_keys(&:to_sym)
|
12
83
|
|
@@ -19,26 +90,27 @@ module Sidekiq
|
|
19
90
|
elsif Sidekiq.server?
|
20
91
|
# Give ourselves plenty of connections. pool is lazy
|
21
92
|
# so we won't create them until we need them.
|
22
|
-
Sidekiq
|
93
|
+
Sidekiq[:concurrency] + 5
|
23
94
|
elsif ENV["RAILS_MAX_THREADS"]
|
24
95
|
Integer(ENV["RAILS_MAX_THREADS"])
|
25
96
|
else
|
26
97
|
5
|
27
98
|
end
|
28
99
|
|
29
|
-
verify_sizing(size, Sidekiq
|
100
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
30
101
|
|
31
102
|
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
103
|
log_info(symbolized_options)
|
33
104
|
|
105
|
+
redis_config = adapter.new(symbolized_options)
|
34
106
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
|
107
|
+
redis_config.new_client
|
36
108
|
end
|
37
109
|
end
|
38
110
|
|
39
111
|
private
|
40
112
|
|
41
|
-
# Sidekiq needs
|
113
|
+
# Sidekiq needs many concurrent Redis connections.
|
42
114
|
#
|
43
115
|
# We need a connection for each Processor.
|
44
116
|
# We need a connection for Pro's real-time change listener
|
@@ -47,48 +119,7 @@ module Sidekiq
|
|
47
119
|
# - enterprise's leader election
|
48
120
|
# - enterprise's cron support
|
49
121
|
def verify_sizing(size, concurrency)
|
50
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
57
|
-
if namespace
|
58
|
-
begin
|
59
|
-
require "redis/namespace"
|
60
|
-
Redis::Namespace.new(namespace, redis: client)
|
61
|
-
rescue LoadError
|
62
|
-
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
63
|
-
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
64
|
-
exit(-127)
|
65
|
-
end
|
66
|
-
else
|
67
|
-
client
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def client_opts(options)
|
72
|
-
opts = options.dup
|
73
|
-
if opts[:namespace]
|
74
|
-
opts.delete(:namespace)
|
75
|
-
end
|
76
|
-
|
77
|
-
if opts[:network_timeout]
|
78
|
-
opts[:timeout] = opts[:network_timeout]
|
79
|
-
opts.delete(:network_timeout)
|
80
|
-
end
|
81
|
-
|
82
|
-
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
83
|
-
|
84
|
-
# Issue #3303, redis-rb will silently retry an operation.
|
85
|
-
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
86
|
-
# is performed twice but I believe this is much, much rarer
|
87
|
-
# than the reconnect silently fixing a problem; we keep it
|
88
|
-
# on by default.
|
89
|
-
opts[:reconnect_attempts] ||= 1
|
90
|
-
|
91
|
-
opts
|
122
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
92
123
|
end
|
93
124
|
|
94
125
|
def log_info(options)
|
@@ -110,9 +141,9 @@ module Sidekiq
|
|
110
141
|
sentinel[:password] = redacted if sentinel[:password]
|
111
142
|
end
|
112
143
|
if Sidekiq.server?
|
113
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
144
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
114
145
|
else
|
115
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
146
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
116
147
|
end
|
117
148
|
end
|
118
149
|
|