sidekiq 4.2.10 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +573 -1
- data/LICENSE +3 -3
- data/README.md +25 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +81 -74
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +585 -285
- data/lib/sidekiq/cli.rb +256 -233
- data/lib/sidekiq/client.rb +86 -83
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +43 -0
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +13 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +282 -0
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +184 -90
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +43 -45
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +176 -91
- data/lib/sidekiq/rails.rb +41 -96
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +117 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +134 -44
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +80 -61
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +129 -86
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +170 -83
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +69 -109
- data/lib/sidekiq/worker.rb +290 -41
- data/lib/sidekiq.rb +185 -77
- data/sidekiq.gemspec +23 -27
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +70 -91
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +364 -144
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +15 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +19 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +5 -19
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +62 -24
- data/web/views/dashboard.erb +24 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +26 -12
- data/web/views/queues.erb +12 -2
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +66 -206
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -9
- data/.gitignore +0 -12
- data/.travis.yml +0 -18
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -173
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -628
- data/Rakefile +0 -12
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -119
- data/lib/sidekiq/exception_handler.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -31
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/lib/sidekiq/util.rb +0 -63
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,123 +1,68 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module Sidekiq
|
3
|
-
def self.hook_rails!
|
4
|
-
return if defined?(@delay_removed)
|
5
|
-
|
6
|
-
ActiveSupport.on_load(:active_record) do
|
7
|
-
include Sidekiq::Extensions::ActiveRecord
|
8
|
-
end
|
9
2
|
|
10
|
-
|
11
|
-
extend Sidekiq::Extensions::ActionMailer
|
12
|
-
end
|
13
|
-
|
14
|
-
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
15
|
-
end
|
16
|
-
|
17
|
-
# Removes the generic aliases which MAY clash with names of already
|
18
|
-
# created methods by other applications. The methods `sidekiq_delay`,
|
19
|
-
# `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
|
20
|
-
def self.remove_delay!
|
21
|
-
@delay_removed = true
|
22
|
-
|
23
|
-
[Extensions::ActiveRecord,
|
24
|
-
Extensions::ActionMailer,
|
25
|
-
Extensions::Klass].each do |mod|
|
26
|
-
mod.module_eval do
|
27
|
-
remove_method :delay if respond_to?(:delay)
|
28
|
-
remove_method :delay_for if respond_to?(:delay_for)
|
29
|
-
remove_method :delay_until if respond_to?(:delay_until)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
3
|
+
require "sidekiq/job"
|
33
4
|
|
5
|
+
module Sidekiq
|
34
6
|
class Rails < ::Rails::Engine
|
35
|
-
|
36
|
-
# change Sidekiq middleware.
|
37
|
-
#
|
38
|
-
# This hook happens after `Rails::Application` is inherited within
|
39
|
-
# config/application.rb and before config is touched, usually within the
|
40
|
-
# class block. Definitely before config/environments/*.rb and
|
41
|
-
# config/initializers/*.rb.
|
42
|
-
config.before_configuration do
|
43
|
-
if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
|
44
|
-
Sidekiq.server_middleware do |chain|
|
45
|
-
require 'sidekiq/middleware/server/active_record'
|
46
|
-
chain.add Sidekiq::Middleware::Server::ActiveRecord
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
initializer 'sidekiq' do
|
52
|
-
Sidekiq.hook_rails!
|
53
|
-
end
|
54
|
-
|
55
|
-
config.after_initialize do
|
56
|
-
# This hook happens after all initializers are run, just before returning
|
57
|
-
# from config/environment.rb back to sidekiq/cli.rb.
|
58
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
59
|
-
# been turned on.
|
60
|
-
#
|
61
|
-
# None of this matters on the client-side, only within the Sidekiq process itself.
|
62
|
-
#
|
63
|
-
Sidekiq.configure_server do |_|
|
64
|
-
if ::Rails::VERSION::MAJOR >= 5
|
65
|
-
# The reloader also takes care of ActiveRecord but is incompatible with
|
66
|
-
# the ActiveRecord middleware so make sure it's not in the chain already.
|
67
|
-
if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
|
68
|
-
raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
|
69
|
-
elsif ::Rails.application.config.cache_classes
|
70
|
-
# The reloader API has proven to be troublesome under load in production.
|
71
|
-
# We won't use it at all when classes are cached, see #3154
|
72
|
-
Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
|
73
|
-
Sidekiq.options[:executor] = Sidekiq::Rails::Executor.new
|
74
|
-
else
|
75
|
-
Sidekiq.logger.debug { "Enabling Rails 5+ live code reloading, so hot!" }
|
76
|
-
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
77
|
-
Psych::Visitors::ToRuby.prepend(Sidekiq::Rails::PsychAutoload)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
class Executor
|
7
|
+
class Reloader
|
84
8
|
def initialize(app = ::Rails.application)
|
85
9
|
@app = app
|
86
10
|
end
|
87
11
|
|
88
12
|
def call
|
89
|
-
@app.
|
13
|
+
@app.reloader.wrap do
|
90
14
|
yield
|
91
15
|
end
|
92
16
|
end
|
93
17
|
|
94
18
|
def inspect
|
95
|
-
"#<Sidekiq::Rails::
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
96
20
|
end
|
97
21
|
end
|
98
22
|
|
99
|
-
|
100
|
-
|
101
|
-
|
23
|
+
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
|
+
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
26
|
+
# manually retried, don't automatically die, etc.
|
27
|
+
#
|
28
|
+
# class SomeJob < ActiveJob::Base
|
29
|
+
# queue_as :default
|
30
|
+
# sidekiq_options retry: 3, backtrace: 10
|
31
|
+
# def perform
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
initializer "sidekiq.active_job_integration" do
|
35
|
+
ActiveSupport.on_load(:active_job) do
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
102
37
|
end
|
38
|
+
end
|
103
39
|
|
104
|
-
|
105
|
-
|
106
|
-
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
107
47
|
end
|
108
48
|
end
|
49
|
+
end
|
109
50
|
|
110
|
-
|
111
|
-
|
112
|
-
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
113
56
|
end
|
114
57
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
58
|
+
# This hook happens after all initializers are run, just before returning
|
59
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
60
|
+
#
|
61
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
62
|
+
config.after_initialize do
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
120
65
|
end
|
121
66
|
end
|
122
|
-
end
|
67
|
+
end
|
123
68
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -1,51 +1,28 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'connection_pool'
|
3
|
-
require 'redis'
|
4
|
-
require 'uri'
|
5
2
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
def create(options={})
|
11
|
-
options = options.symbolize_keys
|
12
|
-
|
13
|
-
options[:url] ||= determine_redis_provider
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis"
|
5
|
+
require "uri"
|
14
6
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
# Sidekiq needs a lot of concurrent Redis connections.
|
30
|
-
#
|
31
|
-
# We need a connection for each Processor.
|
32
|
-
# We need a connection for Pro's real-time change listener
|
33
|
-
# We need a connection to various features to call Redis every few seconds:
|
34
|
-
# - the process heartbeat.
|
35
|
-
# - enterprise's leader election
|
36
|
-
# - enterprise's cron support
|
37
|
-
def verify_sizing(size, concurrency)
|
38
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
|
7
|
+
module Sidekiq
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
39
16
|
end
|
40
17
|
|
41
|
-
def
|
42
|
-
namespace = options[:namespace]
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
43
20
|
|
44
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
45
22
|
if namespace
|
46
23
|
begin
|
47
|
-
require
|
48
|
-
Redis::Namespace.new(namespace, :
|
24
|
+
require "redis/namespace"
|
25
|
+
Redis::Namespace.new(namespace, redis: client)
|
49
26
|
rescue LoadError
|
50
27
|
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
51
28
|
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
@@ -56,6 +33,8 @@ module Sidekiq
|
|
56
33
|
end
|
57
34
|
end
|
58
35
|
|
36
|
+
private
|
37
|
+
|
59
38
|
def client_opts(options)
|
60
39
|
opts = options.dup
|
61
40
|
if opts[:namespace]
|
@@ -67,8 +46,6 @@ module Sidekiq
|
|
67
46
|
opts.delete(:network_timeout)
|
68
47
|
end
|
69
48
|
|
70
|
-
opts[:driver] ||= 'ruby'
|
71
|
-
|
72
49
|
# Issue #3303, redis-rb will silently retry an operation.
|
73
50
|
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
74
51
|
# is performed twice but I believe this is much, much rarer
|
@@ -78,11 +55,81 @@ module Sidekiq
|
|
78
55
|
|
79
56
|
opts
|
80
57
|
end
|
58
|
+
end
|
59
|
+
|
60
|
+
@adapter = RedisAdapter
|
61
|
+
|
62
|
+
class << self
|
63
|
+
attr_reader :adapter
|
64
|
+
|
65
|
+
# RedisConnection.adapter = :redis
|
66
|
+
# RedisConnection.adapter = :redis_client
|
67
|
+
def adapter=(adapter)
|
68
|
+
raise "no" if adapter == self
|
69
|
+
result = case adapter
|
70
|
+
when :redis
|
71
|
+
RedisAdapter
|
72
|
+
when Class
|
73
|
+
adapter
|
74
|
+
else
|
75
|
+
require "sidekiq/#{adapter}_adapter"
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
@adapter = result if result
|
79
|
+
end
|
80
|
+
|
81
|
+
def create(options = {})
|
82
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
83
|
+
|
84
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
85
|
+
symbolized_options[:url] = u
|
86
|
+
end
|
87
|
+
|
88
|
+
size = if symbolized_options[:size]
|
89
|
+
symbolized_options[:size]
|
90
|
+
elsif Sidekiq.server?
|
91
|
+
# Give ourselves plenty of connections. pool is lazy
|
92
|
+
# so we won't create them until we need them.
|
93
|
+
Sidekiq[:concurrency] + 5
|
94
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
95
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
96
|
+
else
|
97
|
+
5
|
98
|
+
end
|
99
|
+
|
100
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
101
|
+
|
102
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
103
|
+
log_info(symbolized_options)
|
104
|
+
|
105
|
+
redis_config = adapter.new(symbolized_options)
|
106
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
107
|
+
redis_config.new_client
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
# Sidekiq needs many concurrent Redis connections.
|
114
|
+
#
|
115
|
+
# We need a connection for each Processor.
|
116
|
+
# We need a connection for Pro's real-time change listener
|
117
|
+
# We need a connection to various features to call Redis every few seconds:
|
118
|
+
# - the process heartbeat.
|
119
|
+
# - enterprise's leader election
|
120
|
+
# - enterprise's cron support
|
121
|
+
def verify_sizing(size, concurrency)
|
122
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
123
|
+
end
|
81
124
|
|
82
125
|
def log_info(options)
|
83
|
-
# Don't log Redis AUTH password
|
84
126
|
redacted = "REDACTED"
|
85
|
-
|
127
|
+
|
128
|
+
# Deep clone so we can muck with these options all we want and exclude
|
129
|
+
# params from dump-and-load that may contain objects that Marshal is
|
130
|
+
# unable to safely dump.
|
131
|
+
keys = options.keys - [:logger, :ssl_params]
|
132
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
86
133
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
87
134
|
uri.password = redacted
|
88
135
|
scrubbed_options[:url] = uri.to_s
|
@@ -90,17 +137,39 @@ module Sidekiq
|
|
90
137
|
if scrubbed_options[:password]
|
91
138
|
scrubbed_options[:password] = redacted
|
92
139
|
end
|
140
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
141
|
+
sentinel[:password] = redacted if sentinel[:password]
|
142
|
+
end
|
93
143
|
if Sidekiq.server?
|
94
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
144
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
95
145
|
else
|
96
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
146
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
97
147
|
end
|
98
148
|
end
|
99
149
|
|
100
150
|
def determine_redis_provider
|
101
|
-
|
102
|
-
|
151
|
+
# If you have this in your environment:
|
152
|
+
# MY_REDIS_URL=redis://hostname.example.com:1238/4
|
153
|
+
# then set:
|
154
|
+
# REDIS_PROVIDER=MY_REDIS_URL
|
155
|
+
# and Sidekiq will find your custom URL variable with no custom
|
156
|
+
# initialization code at all.
|
157
|
+
#
|
158
|
+
p = ENV["REDIS_PROVIDER"]
|
159
|
+
if p && p =~ /:/
|
160
|
+
raise <<~EOM
|
161
|
+
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
162
|
+
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
163
|
+
|
164
|
+
REDISTOGO_URL=redis://somehost.example.com:6379/4
|
165
|
+
REDIS_PROVIDER=REDISTOGO_URL
|
166
|
+
EOM
|
167
|
+
end
|
103
168
|
|
169
|
+
ENV[
|
170
|
+
p || "REDIS_URL"
|
171
|
+
]
|
172
|
+
end
|
104
173
|
end
|
105
174
|
end
|
106
175
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|