sidekiq 6.0.7 → 6.5.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +209 -2
- data/LICENSE +3 -3
- data/README.md +11 -10
- data/bin/sidekiq +8 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +180 -123
- data/lib/sidekiq/cli.rb +80 -45
- data/lib/sidekiq/client.rb +52 -71
- data/lib/sidekiq/{util.rb → component.rb} +11 -14
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +41 -30
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +36 -36
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +123 -63
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +28 -17
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +19 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +41 -41
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +87 -53
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +39 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +40 -34
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +136 -16
- data/lib/sidekiq.rb +107 -30
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +113 -65
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +42 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +51 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +29 -51
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
@@ -0,0 +1,19 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module ServerMiddleware
|
3
|
+
attr_accessor :config
|
4
|
+
def redis_pool
|
5
|
+
config.redis_pool
|
6
|
+
end
|
7
|
+
|
8
|
+
def logger
|
9
|
+
config.logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def redis(&block)
|
13
|
+
config.redis(&block)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# no difference for now
|
18
|
+
ClientMiddleware = ServerMiddleware
|
19
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
data/lib/sidekiq/paginator.rb
CHANGED
@@ -16,22 +16,22 @@ module Sidekiq
|
|
16
16
|
|
17
17
|
case type
|
18
18
|
when "zset"
|
19
|
-
total_size, items = conn.multi {
|
20
|
-
|
19
|
+
total_size, items = conn.multi { |transaction|
|
20
|
+
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
|
22
|
+
transaction.zrevrange(key, starting, ending, withscores: true)
|
23
23
|
else
|
24
|
-
|
24
|
+
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
28
28
|
when "list"
|
29
|
-
total_size, items = conn.multi {
|
30
|
-
|
29
|
+
total_size, items = conn.multi { |transaction|
|
30
|
+
transaction.llen(key)
|
31
31
|
if rev
|
32
|
-
|
32
|
+
transaction.lrange(key, -ending - 1, -starting - 1)
|
33
33
|
else
|
34
|
-
|
34
|
+
transaction.lrange(key, starting, ending)
|
35
35
|
end
|
36
36
|
}
|
37
37
|
items.reverse! if rev
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
30
29
|
|
31
|
-
def initialize(
|
32
|
-
@
|
30
|
+
def initialize(options, &block)
|
31
|
+
@callback = block
|
33
32
|
@down = false
|
34
33
|
@done = false
|
35
34
|
@job = nil
|
36
35
|
@thread = nil
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@
|
40
|
-
@
|
36
|
+
@config = options
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
+
@retrier = Sidekiq::JobRetry.new(options)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -66,26 +66,26 @@ module Sidekiq
|
|
66
66
|
|
67
67
|
def run
|
68
68
|
process_one until @done
|
69
|
-
@
|
69
|
+
@callback.call(self)
|
70
70
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
71
|
+
@callback.call(self)
|
72
72
|
rescue Exception => ex
|
73
|
-
@
|
73
|
+
@callback.call(self, ex)
|
74
74
|
end
|
75
75
|
|
76
|
-
def process_one
|
76
|
+
def process_one(&block)
|
77
77
|
@job = fetch
|
78
78
|
process(@job) if @job
|
79
79
|
@job = nil
|
80
80
|
end
|
81
81
|
|
82
82
|
def get_one
|
83
|
-
|
83
|
+
uow = @strategy.retrieve_work
|
84
84
|
if @down
|
85
85
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
86
|
@down = nil
|
87
87
|
end
|
88
|
-
|
88
|
+
uow
|
89
89
|
rescue Sidekiq::Shutdown
|
90
90
|
rescue => ex
|
91
91
|
handle_fetch_exception(ex)
|
@@ -130,10 +130,10 @@ module Sidekiq
|
|
130
130
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
131
|
@reloader.call do
|
132
132
|
klass = constantize(job_hash["class"])
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
133
|
+
inst = klass.new
|
134
|
+
inst.jid = job_hash["jid"]
|
135
|
+
@retrier.local(inst, jobstr, queue) do
|
136
|
+
yield inst
|
137
137
|
end
|
138
138
|
end
|
139
139
|
end
|
@@ -142,9 +142,9 @@ module Sidekiq
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
def process(
|
146
|
-
jobstr =
|
147
|
-
queue =
|
145
|
+
def process(uow)
|
146
|
+
jobstr = uow.job
|
147
|
+
queue = uow.queue_name
|
148
148
|
|
149
149
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
150
|
job_hash = nil
|
@@ -154,14 +154,14 @@ module Sidekiq
|
|
154
154
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
155
|
# we can't notify because the job isn't a valid hash payload.
|
156
156
|
DeadSet.new.kill(jobstr, notify_failure: false)
|
157
|
-
return
|
157
|
+
return uow.acknowledge
|
158
158
|
end
|
159
159
|
|
160
160
|
ack = false
|
161
161
|
begin
|
162
|
-
dispatch(job_hash, queue, jobstr) do |
|
163
|
-
|
164
|
-
execute_job(
|
162
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
163
|
+
@config.server_middleware.invoke(inst, job_hash, queue) do
|
164
|
+
execute_job(inst, job_hash["args"])
|
165
165
|
end
|
166
166
|
end
|
167
167
|
ack = true
|
@@ -186,14 +186,14 @@ module Sidekiq
|
|
186
186
|
if ack
|
187
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
188
|
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
|
189
|
+
uow.acknowledge
|
190
190
|
end
|
191
191
|
end
|
192
192
|
end
|
193
193
|
end
|
194
194
|
|
195
|
-
def execute_job(
|
196
|
-
|
195
|
+
def execute_job(inst, cloned_args)
|
196
|
+
inst.perform(*cloned_args)
|
197
197
|
end
|
198
198
|
|
199
199
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +219,39 @@ module Sidekiq
|
|
219
219
|
end
|
220
220
|
|
221
221
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
222
|
+
class SharedWorkState
|
223
223
|
def initialize
|
224
|
-
@
|
224
|
+
@work_state = {}
|
225
225
|
@lock = Mutex.new
|
226
226
|
end
|
227
227
|
|
228
228
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
229
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
230
|
end
|
231
231
|
|
232
232
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
233
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
234
|
end
|
235
235
|
|
236
236
|
def dup
|
237
|
-
@lock.synchronize { @
|
237
|
+
@lock.synchronize { @work_state.dup }
|
238
238
|
end
|
239
239
|
|
240
240
|
def size
|
241
|
-
@lock.synchronize { @
|
241
|
+
@lock.synchronize { @work_state.size }
|
242
242
|
end
|
243
243
|
|
244
244
|
def clear
|
245
|
-
@lock.synchronize { @
|
245
|
+
@lock.synchronize { @work_state.clear }
|
246
246
|
end
|
247
247
|
end
|
248
248
|
|
249
249
|
PROCESSED = Counter.new
|
250
250
|
FAILURE = Counter.new
|
251
|
-
|
251
|
+
WORK_STATE = SharedWorkState.new
|
252
252
|
|
253
253
|
def stats(jobstr, queue)
|
254
|
-
|
254
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
255
|
|
256
256
|
begin
|
257
257
|
yield
|
@@ -259,7 +259,7 @@ module Sidekiq
|
|
259
259
|
FAILURE.incr
|
260
260
|
raise
|
261
261
|
ensure
|
262
|
-
|
262
|
+
WORK_STATE.delete(tid)
|
263
263
|
PROCESSED.incr
|
264
264
|
end
|
265
265
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,9 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -17,35 +33,35 @@ module Sidekiq
|
|
17
33
|
# end
|
18
34
|
initializer "sidekiq.active_job_integration" do
|
19
35
|
ActiveSupport.on_load(:active_job) do
|
20
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
|
+
end
|
21
48
|
end
|
22
49
|
end
|
23
50
|
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
24
58
|
# This hook happens after all initializers are run, just before returning
|
25
59
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
60
|
#
|
29
61
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
62
|
config.after_initialize do
|
31
|
-
Sidekiq.configure_server do |
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
49
65
|
end
|
50
66
|
end
|
51
67
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,57 +5,20 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
9
|
-
class
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
options[:url] = u
|
17
|
-
end
|
18
|
-
|
19
|
-
size = if options[:size]
|
20
|
-
options[:size]
|
21
|
-
elsif Sidekiq.server?
|
22
|
-
# Give ourselves plenty of connections. pool is lazy
|
23
|
-
# so we won't create them until we need them.
|
24
|
-
Sidekiq.options[:concurrency] + 5
|
25
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
26
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
27
|
-
else
|
28
|
-
5
|
29
|
-
end
|
30
|
-
|
31
|
-
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
32
|
-
|
33
|
-
pool_timeout = options[:pool_timeout] || 1
|
34
|
-
log_info(options)
|
35
|
-
|
36
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
37
|
-
build_client(options)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
private
|
42
|
-
|
43
|
-
# Sidekiq needs a lot of concurrent Redis connections.
|
44
|
-
#
|
45
|
-
# We need a connection for each Processor.
|
46
|
-
# We need a connection for Pro's real-time change listener
|
47
|
-
# We need a connection to various features to call Redis every few seconds:
|
48
|
-
# - the process heartbeat.
|
49
|
-
# - enterprise's leader election
|
50
|
-
# - enterprise's cron support
|
51
|
-
def verify_sizing(size, concurrency)
|
52
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
53
16
|
end
|
54
17
|
|
55
|
-
def
|
56
|
-
namespace = options[:namespace]
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
57
20
|
|
58
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
59
22
|
if namespace
|
60
23
|
begin
|
61
24
|
require "redis/namespace"
|
@@ -70,6 +33,8 @@ module Sidekiq
|
|
70
33
|
end
|
71
34
|
end
|
72
35
|
|
36
|
+
private
|
37
|
+
|
73
38
|
def client_opts(options)
|
74
39
|
opts = options.dup
|
75
40
|
if opts[:namespace]
|
@@ -92,12 +57,81 @@ module Sidekiq
|
|
92
57
|
|
93
58
|
opts
|
94
59
|
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@adapter = RedisAdapter
|
63
|
+
|
64
|
+
class << self
|
65
|
+
attr_reader :adapter
|
66
|
+
|
67
|
+
# RedisConnection.adapter = :redis
|
68
|
+
# RedisConnection.adapter = :redis_client
|
69
|
+
def adapter=(adapter)
|
70
|
+
raise "no" if adapter == self
|
71
|
+
result = case adapter
|
72
|
+
when :redis
|
73
|
+
RedisAdapter
|
74
|
+
when Class
|
75
|
+
adapter
|
76
|
+
else
|
77
|
+
require "sidekiq/#{adapter}_adapter"
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
@adapter = result if result
|
81
|
+
end
|
82
|
+
|
83
|
+
def create(options = {})
|
84
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
85
|
+
|
86
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
87
|
+
symbolized_options[:url] = u
|
88
|
+
end
|
89
|
+
|
90
|
+
size = if symbolized_options[:size]
|
91
|
+
symbolized_options[:size]
|
92
|
+
elsif Sidekiq.server?
|
93
|
+
# Give ourselves plenty of connections. pool is lazy
|
94
|
+
# so we won't create them until we need them.
|
95
|
+
Sidekiq[:concurrency] + 5
|
96
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
97
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
98
|
+
else
|
99
|
+
5
|
100
|
+
end
|
101
|
+
|
102
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
103
|
+
|
104
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
105
|
+
log_info(symbolized_options)
|
106
|
+
|
107
|
+
redis_config = adapter.new(symbolized_options)
|
108
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
109
|
+
redis_config.new_client
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
# Sidekiq needs many concurrent Redis connections.
|
116
|
+
#
|
117
|
+
# We need a connection for each Processor.
|
118
|
+
# We need a connection for Pro's real-time change listener
|
119
|
+
# We need a connection to various features to call Redis every few seconds:
|
120
|
+
# - the process heartbeat.
|
121
|
+
# - enterprise's leader election
|
122
|
+
# - enterprise's cron support
|
123
|
+
def verify_sizing(size, concurrency)
|
124
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
125
|
+
end
|
95
126
|
|
96
127
|
def log_info(options)
|
97
128
|
redacted = "REDACTED"
|
98
129
|
|
99
|
-
#
|
100
|
-
|
130
|
+
# Deep clone so we can muck with these options all we want and exclude
|
131
|
+
# params from dump-and-load that may contain objects that Marshal is
|
132
|
+
# unable to safely dump.
|
133
|
+
keys = options.keys - [:logger, :ssl_params]
|
134
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
101
135
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
102
136
|
uri.password = redacted
|
103
137
|
scrubbed_options[:url] = uri.to_s
|
@@ -109,9 +143,9 @@ module Sidekiq
|
|
109
143
|
sentinel[:password] = redacted if sentinel[:password]
|
110
144
|
end
|
111
145
|
if Sidekiq.server?
|
112
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
146
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
113
147
|
else
|
114
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
148
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
115
149
|
end
|
116
150
|
end
|
117
151
|
|
@@ -124,7 +158,7 @@ module Sidekiq
|
|
124
158
|
# initialization code at all.
|
125
159
|
#
|
126
160
|
p = ENV["REDIS_PROVIDER"]
|
127
|
-
if p && p =~
|
161
|
+
if p && p =~ /:/
|
128
162
|
raise <<~EOM
|
129
163
|
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
130
164
|
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|