sidekiq 6.4.2 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +24 -0
- data/bin/sidekiqload +15 -3
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +47 -20
- data/lib/sidekiq/cli.rb +33 -32
- data/lib/sidekiq/client.rb +4 -4
- data/lib/sidekiq/component.rb +64 -0
- data/lib/sidekiq/delay.rb +1 -1
- data/lib/sidekiq/fetch.rb +16 -14
- data/lib/sidekiq/job_retry.rb +6 -5
- data/lib/sidekiq/job_util.rb +7 -3
- data/lib/sidekiq/launcher.rb +18 -17
- data/lib/sidekiq/logger.rb +1 -1
- data/lib/sidekiq/manager.rb +23 -20
- data/lib/sidekiq/middleware/chain.rb +20 -11
- data/lib/sidekiq/middleware/current_attributes.rb +4 -0
- data/lib/sidekiq/middleware/i18n.rb +2 -0
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/paginator.rb +2 -2
- data/lib/sidekiq/processor.rb +12 -12
- data/lib/sidekiq/rails.rb +5 -5
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -47
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +11 -10
- data/lib/sidekiq/testing.rb +1 -1
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/helpers.rb +1 -1
- data/lib/sidekiq/worker.rb +2 -1
- data/lib/sidekiq.rb +78 -17
- data/web/locales/pt-br.yml +27 -9
- metadata +8 -4
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,8 +5,81 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
16
|
+
end
|
17
|
+
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
20
|
+
|
21
|
+
client = Redis.new client_opts(@options)
|
22
|
+
if namespace
|
23
|
+
begin
|
24
|
+
require "redis/namespace"
|
25
|
+
Redis::Namespace.new(namespace, redis: client)
|
26
|
+
rescue LoadError
|
27
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
28
|
+
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
29
|
+
exit(-127)
|
30
|
+
end
|
31
|
+
else
|
32
|
+
client
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def client_opts(options)
|
39
|
+
opts = options.dup
|
40
|
+
if opts[:namespace]
|
41
|
+
opts.delete(:namespace)
|
42
|
+
end
|
43
|
+
|
44
|
+
if opts[:network_timeout]
|
45
|
+
opts[:timeout] = opts[:network_timeout]
|
46
|
+
opts.delete(:network_timeout)
|
47
|
+
end
|
48
|
+
|
49
|
+
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
50
|
+
|
51
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
52
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
53
|
+
# is performed twice but I believe this is much, much rarer
|
54
|
+
# than the reconnect silently fixing a problem; we keep it
|
55
|
+
# on by default.
|
56
|
+
opts[:reconnect_attempts] ||= 1
|
57
|
+
|
58
|
+
opts
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@adapter = RedisAdapter
|
63
|
+
|
9
64
|
class << self
|
65
|
+
attr_reader :adapter
|
66
|
+
|
67
|
+
# RedisConnection.adapter = :redis
|
68
|
+
# RedisConnection.adapter = :redis_client
|
69
|
+
def adapter=(adapter)
|
70
|
+
raise "no" if adapter == self
|
71
|
+
result = case adapter
|
72
|
+
when :redis
|
73
|
+
RedisAdapter
|
74
|
+
when Class
|
75
|
+
adapter
|
76
|
+
else
|
77
|
+
require "sidekiq/#{adapter}_adapter"
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
@adapter = result if result
|
81
|
+
end
|
82
|
+
|
10
83
|
def create(options = {})
|
11
84
|
symbolized_options = options.transform_keys(&:to_sym)
|
12
85
|
|
@@ -19,20 +92,21 @@ module Sidekiq
|
|
19
92
|
elsif Sidekiq.server?
|
20
93
|
# Give ourselves plenty of connections. pool is lazy
|
21
94
|
# so we won't create them until we need them.
|
22
|
-
Sidekiq
|
95
|
+
Sidekiq[:concurrency] + 5
|
23
96
|
elsif ENV["RAILS_MAX_THREADS"]
|
24
97
|
Integer(ENV["RAILS_MAX_THREADS"])
|
25
98
|
else
|
26
99
|
5
|
27
100
|
end
|
28
101
|
|
29
|
-
verify_sizing(size, Sidekiq
|
102
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
30
103
|
|
31
104
|
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
105
|
log_info(symbolized_options)
|
33
106
|
|
107
|
+
redis_config = adapter.new(symbolized_options)
|
34
108
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
|
109
|
+
redis_config.new_client
|
36
110
|
end
|
37
111
|
end
|
38
112
|
|
@@ -50,47 +124,6 @@ module Sidekiq
|
|
50
124
|
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
51
125
|
end
|
52
126
|
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
57
|
-
if namespace
|
58
|
-
begin
|
59
|
-
require "redis/namespace"
|
60
|
-
Redis::Namespace.new(namespace, redis: client)
|
61
|
-
rescue LoadError
|
62
|
-
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
63
|
-
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
64
|
-
exit(-127)
|
65
|
-
end
|
66
|
-
else
|
67
|
-
client
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def client_opts(options)
|
72
|
-
opts = options.dup
|
73
|
-
if opts[:namespace]
|
74
|
-
opts.delete(:namespace)
|
75
|
-
end
|
76
|
-
|
77
|
-
if opts[:network_timeout]
|
78
|
-
opts[:timeout] = opts[:network_timeout]
|
79
|
-
opts.delete(:network_timeout)
|
80
|
-
end
|
81
|
-
|
82
|
-
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
83
|
-
|
84
|
-
# Issue #3303, redis-rb will silently retry an operation.
|
85
|
-
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
86
|
-
# is performed twice but I believe this is much, much rarer
|
87
|
-
# than the reconnect silently fixing a problem; we keep it
|
88
|
-
# on by default.
|
89
|
-
opts[:reconnect_attempts] ||= 1
|
90
|
-
|
91
|
-
opts
|
92
|
-
end
|
93
|
-
|
94
127
|
def log_info(options)
|
95
128
|
redacted = "REDACTED"
|
96
129
|
|
@@ -110,9 +143,9 @@ module Sidekiq
|
|
110
143
|
sentinel[:password] = redacted if sentinel[:password]
|
111
144
|
end
|
112
145
|
if Sidekiq.server?
|
113
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
146
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
114
147
|
else
|
115
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
148
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
116
149
|
end
|
117
150
|
end
|
118
151
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/util"
|
5
4
|
require "sidekiq/api"
|
5
|
+
require "sidekiq/component"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
module Scheduled
|
@@ -52,8 +52,8 @@ module Sidekiq
|
|
52
52
|
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
53
53
|
end
|
54
54
|
|
55
|
-
conn.evalsha(@lua_zpopbyscore_sha, keys
|
56
|
-
rescue
|
55
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
|
56
|
+
rescue RedisConnection.adapter::CommandError => e
|
57
57
|
raise unless e.message.start_with?("NOSCRIPT")
|
58
58
|
|
59
59
|
@lua_zpopbyscore_sha = nil
|
@@ -67,12 +67,13 @@ module Sidekiq
|
|
67
67
|
# just pops the job back onto its original queue so the
|
68
68
|
# workers can pick it up like any other job.
|
69
69
|
class Poller
|
70
|
-
include
|
70
|
+
include Sidekiq::Component
|
71
71
|
|
72
72
|
INITIAL_WAIT = 10
|
73
73
|
|
74
|
-
def initialize
|
75
|
-
@
|
74
|
+
def initialize(options)
|
75
|
+
@config = options
|
76
|
+
@enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
|
76
77
|
@sleeper = ConnectionPool::TimedStack.new
|
77
78
|
@done = false
|
78
79
|
@thread = nil
|
@@ -100,7 +101,7 @@ module Sidekiq
|
|
100
101
|
enqueue
|
101
102
|
wait
|
102
103
|
end
|
103
|
-
|
104
|
+
logger.info("Scheduler exiting...")
|
104
105
|
}
|
105
106
|
end
|
106
107
|
|
@@ -171,14 +172,14 @@ module Sidekiq
|
|
171
172
|
#
|
172
173
|
# We only do this if poll_interval_average is unset (the default).
|
173
174
|
def poll_interval_average
|
174
|
-
|
175
|
+
@config[:poll_interval_average] ||= scaled_poll_interval
|
175
176
|
end
|
176
177
|
|
177
178
|
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
178
179
|
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
179
180
|
# Redis if you run many Sidekiq processes.
|
180
181
|
def scaled_poll_interval
|
181
|
-
process_count *
|
182
|
+
process_count * @config[:average_scheduled_poll_interval]
|
182
183
|
end
|
183
184
|
|
184
185
|
def process_count
|
@@ -197,7 +198,7 @@ module Sidekiq
|
|
197
198
|
# to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
|
198
199
|
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
199
200
|
total = 0
|
200
|
-
total += INITIAL_WAIT unless
|
201
|
+
total += INITIAL_WAIT unless @config[:poll_interval_average]
|
201
202
|
total += (5 * rand)
|
202
203
|
|
203
204
|
@sleeper.pop(total)
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/client"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class TransactionAwareClient
|
8
|
+
def initialize(redis_pool)
|
9
|
+
@redis_client = Client.new(redis_pool)
|
10
|
+
end
|
11
|
+
|
12
|
+
def push(item)
|
13
|
+
# pre-allocate the JID so we can return it immediately and
|
14
|
+
# save it to the database as part of the transaction.
|
15
|
+
item["jid"] ||= SecureRandom.hex(12)
|
16
|
+
AfterCommitEverywhere.after_commit { @redis_client.push(item) }
|
17
|
+
item["jid"]
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# We don't provide transactionality for push_bulk because we don't want
|
22
|
+
# to hold potentially hundreds of thousands of job records in memory due to
|
23
|
+
# a long running enqueue process.
|
24
|
+
def push_bulk(items)
|
25
|
+
@redis_client.push_bulk(items)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
|
32
|
+
module Sidekiq
|
33
|
+
def self.transactional_push!
|
34
|
+
begin
|
35
|
+
require "after_commit_everywhere"
|
36
|
+
rescue LoadError
|
37
|
+
Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
|
38
|
+
raise
|
39
|
+
end
|
40
|
+
|
41
|
+
default_job_options["client_class"] = Sidekiq::TransactionAwareClient
|
42
|
+
Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
|
43
|
+
true
|
44
|
+
end
|
45
|
+
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -301,7 +301,7 @@ module Sidekiq
|
|
301
301
|
end
|
302
302
|
|
303
303
|
def environment_title_prefix
|
304
|
-
environment = Sidekiq
|
304
|
+
environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
305
305
|
|
306
306
|
"[#{environment.upcase}] " unless environment == "production"
|
307
307
|
end
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -359,7 +359,8 @@ module Sidekiq
|
|
359
359
|
|
360
360
|
def build_client # :nodoc:
|
361
361
|
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
362
|
-
Sidekiq::Client
|
362
|
+
client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
|
363
|
+
client_class.new(pool)
|
363
364
|
end
|
364
365
|
end
|
365
366
|
end
|