sidekiq 5.2.10 → 6.5.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +79 -67
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +504 -307
- data/lib/sidekiq/cli.rb +190 -206
- data/lib/sidekiq/client.rb +77 -81
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +8 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +126 -106
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +177 -83
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +104 -97
- data/lib/sidekiq/rails.rb +47 -37
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +108 -77
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +64 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +68 -58
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +100 -77
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +134 -94
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +65 -105
- data/lib/sidekiq/worker.rb +253 -106
- data/lib/sidekiq.rb +170 -62
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +53 -89
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +53 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -19
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,38 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "sidekiq/job"
|
4
|
+
|
3
5
|
module Sidekiq
|
4
6
|
class Rails < ::Rails::Engine
|
5
|
-
# We need to setup this up before any application configuration which might
|
6
|
-
# change Sidekiq middleware.
|
7
|
-
#
|
8
|
-
# This hook happens after `Rails::Application` is inherited within
|
9
|
-
# config/application.rb and before config is touched, usually within the
|
10
|
-
# class block. Definitely before config/environments/*.rb and
|
11
|
-
# config/initializers/*.rb.
|
12
|
-
config.before_configuration do
|
13
|
-
if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
|
14
|
-
Sidekiq.server_middleware do |chain|
|
15
|
-
require 'sidekiq/middleware/server/active_record'
|
16
|
-
chain.add Sidekiq::Middleware::Server::ActiveRecord
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
config.after_initialize do
|
22
|
-
# This hook happens after all initializers are run, just before returning
|
23
|
-
# from config/environment.rb back to sidekiq/cli.rb.
|
24
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
25
|
-
# been turned on.
|
26
|
-
#
|
27
|
-
# None of this matters on the client-side, only within the Sidekiq process itself.
|
28
|
-
#
|
29
|
-
Sidekiq.configure_server do |_|
|
30
|
-
if ::Rails::VERSION::MAJOR >= 5
|
31
|
-
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
7
|
class Reloader
|
37
8
|
def initialize(app = ::Rails.application)
|
38
9
|
@app = app
|
@@ -48,11 +19,50 @@ module Sidekiq
|
|
48
19
|
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
20
|
end
|
50
21
|
end
|
51
|
-
end if defined?(::Rails)
|
52
|
-
end
|
53
22
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
23
|
+
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
|
+
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
26
|
+
# manually retried, don't automatically die, etc.
|
27
|
+
#
|
28
|
+
# class SomeJob < ActiveJob::Base
|
29
|
+
# queue_as :default
|
30
|
+
# sidekiq_options retry: 3, backtrace: 10
|
31
|
+
# def perform
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
initializer "sidekiq.active_job_integration" do
|
35
|
+
ActiveSupport.on_load(:active_job) do
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
58
|
+
# This hook happens after all initializers are run, just before returning
|
59
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
60
|
+
#
|
61
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
62
|
+
config.after_initialize do
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
58
68
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -1,62 +1,28 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'connection_pool'
|
3
|
-
require 'redis'
|
4
|
-
require 'uri'
|
5
2
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
def create(options={})
|
11
|
-
options.keys.each do |key|
|
12
|
-
options[key.to_sym] = options.delete(key)
|
13
|
-
end
|
14
|
-
|
15
|
-
options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
|
16
|
-
options[:url] ||= determine_redis_provider
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis"
|
5
|
+
require "uri"
|
17
6
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
29
|
-
|
30
|
-
pool_timeout = options[:pool_timeout] || 1
|
31
|
-
log_info(options)
|
32
|
-
|
33
|
-
ConnectionPool.new(:timeout => pool_timeout, :size => size) do
|
34
|
-
build_client(options)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
private
|
39
|
-
|
40
|
-
# Sidekiq needs a lot of concurrent Redis connections.
|
41
|
-
#
|
42
|
-
# We need a connection for each Processor.
|
43
|
-
# We need a connection for Pro's real-time change listener
|
44
|
-
# We need a connection to various features to call Redis every few seconds:
|
45
|
-
# - the process heartbeat.
|
46
|
-
# - enterprise's leader election
|
47
|
-
# - enterprise's cron support
|
48
|
-
def verify_sizing(size, concurrency)
|
49
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
|
7
|
+
module Sidekiq
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
50
16
|
end
|
51
17
|
|
52
|
-
def
|
53
|
-
namespace = options[:namespace]
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
54
20
|
|
55
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
56
22
|
if namespace
|
57
23
|
begin
|
58
|
-
require
|
59
|
-
Redis::Namespace.new(namespace, :
|
24
|
+
require "redis/namespace"
|
25
|
+
Redis::Namespace.new(namespace, redis: client)
|
60
26
|
rescue LoadError
|
61
27
|
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
62
28
|
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
@@ -67,6 +33,8 @@ module Sidekiq
|
|
67
33
|
end
|
68
34
|
end
|
69
35
|
|
36
|
+
private
|
37
|
+
|
70
38
|
def client_opts(options)
|
71
39
|
opts = options.dup
|
72
40
|
if opts[:namespace]
|
@@ -78,8 +46,6 @@ module Sidekiq
|
|
78
46
|
opts.delete(:network_timeout)
|
79
47
|
end
|
80
48
|
|
81
|
-
opts[:driver] ||= Redis::Connection.drivers.last || 'ruby'
|
82
|
-
|
83
49
|
# Issue #3303, redis-rb will silently retry an operation.
|
84
50
|
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
85
51
|
# is performed twice but I believe this is much, much rarer
|
@@ -89,11 +55,81 @@ module Sidekiq
|
|
89
55
|
|
90
56
|
opts
|
91
57
|
end
|
58
|
+
end
|
59
|
+
|
60
|
+
@adapter = RedisAdapter
|
61
|
+
|
62
|
+
class << self
|
63
|
+
attr_reader :adapter
|
64
|
+
|
65
|
+
# RedisConnection.adapter = :redis
|
66
|
+
# RedisConnection.adapter = :redis_client
|
67
|
+
def adapter=(adapter)
|
68
|
+
raise "no" if adapter == self
|
69
|
+
result = case adapter
|
70
|
+
when :redis
|
71
|
+
RedisAdapter
|
72
|
+
when Class
|
73
|
+
adapter
|
74
|
+
else
|
75
|
+
require "sidekiq/#{adapter}_adapter"
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
@adapter = result if result
|
79
|
+
end
|
80
|
+
|
81
|
+
def create(options = {})
|
82
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
83
|
+
|
84
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
85
|
+
symbolized_options[:url] = u
|
86
|
+
end
|
87
|
+
|
88
|
+
size = if symbolized_options[:size]
|
89
|
+
symbolized_options[:size]
|
90
|
+
elsif Sidekiq.server?
|
91
|
+
# Give ourselves plenty of connections. pool is lazy
|
92
|
+
# so we won't create them until we need them.
|
93
|
+
Sidekiq[:concurrency] + 5
|
94
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
95
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
96
|
+
else
|
97
|
+
5
|
98
|
+
end
|
99
|
+
|
100
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
101
|
+
|
102
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
103
|
+
log_info(symbolized_options)
|
104
|
+
|
105
|
+
redis_config = adapter.new(symbolized_options)
|
106
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
107
|
+
redis_config.new_client
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
# Sidekiq needs many concurrent Redis connections.
|
114
|
+
#
|
115
|
+
# We need a connection for each Processor.
|
116
|
+
# We need a connection for Pro's real-time change listener
|
117
|
+
# We need a connection to various features to call Redis every few seconds:
|
118
|
+
# - the process heartbeat.
|
119
|
+
# - enterprise's leader election
|
120
|
+
# - enterprise's cron support
|
121
|
+
def verify_sizing(size, concurrency)
|
122
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
123
|
+
end
|
92
124
|
|
93
125
|
def log_info(options)
|
94
|
-
# Don't log Redis AUTH password
|
95
126
|
redacted = "REDACTED"
|
96
|
-
|
127
|
+
|
128
|
+
# Deep clone so we can muck with these options all we want and exclude
|
129
|
+
# params from dump-and-load that may contain objects that Marshal is
|
130
|
+
# unable to safely dump.
|
131
|
+
keys = options.keys - [:logger, :ssl_params]
|
132
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
97
133
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
98
134
|
uri.password = redacted
|
99
135
|
scrubbed_options[:url] = uri.to_s
|
@@ -101,10 +137,13 @@ module Sidekiq
|
|
101
137
|
if scrubbed_options[:password]
|
102
138
|
scrubbed_options[:password] = redacted
|
103
139
|
end
|
140
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
141
|
+
sentinel[:password] = redacted if sentinel[:password]
|
142
|
+
end
|
104
143
|
if Sidekiq.server?
|
105
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
144
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
106
145
|
else
|
107
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
146
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
108
147
|
end
|
109
148
|
end
|
110
149
|
|
@@ -115,30 +154,22 @@ module Sidekiq
|
|
115
154
|
# REDIS_PROVIDER=MY_REDIS_URL
|
116
155
|
# and Sidekiq will find your custom URL variable with no custom
|
117
156
|
# initialization code at all.
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
REDISTOGO_URL=redis://somehost.example.com:6379/4
|
129
|
-
|
130
|
-
Use REDIS_URL if you wish to point Sidekiq to a URL directly.
|
131
|
-
|
132
|
-
This configuration error will crash starting in Sidekiq 5.3.
|
133
|
-
|
134
|
-
#################################################################################
|
135
|
-
EOM
|
157
|
+
#
|
158
|
+
p = ENV["REDIS_PROVIDER"]
|
159
|
+
if p && p =~ /:/
|
160
|
+
raise <<~EOM
|
161
|
+
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
162
|
+
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
163
|
+
|
164
|
+
REDISTOGO_URL=redis://somehost.example.com:6379/4
|
165
|
+
REDIS_PROVIDER=REDISTOGO_URL
|
166
|
+
EOM
|
136
167
|
end
|
168
|
+
|
137
169
|
ENV[
|
138
|
-
|
170
|
+
p || "REDIS_URL"
|
139
171
|
]
|
140
172
|
end
|
141
|
-
|
142
173
|
end
|
143
174
|
end
|
144
175
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|