sidekiq 6.2.2 → 6.5.5
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +150 -1
- data/LICENSE +3 -3
- data/README.md +9 -4
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +232 -94
- data/lib/sidekiq/cli.rb +60 -40
- data/lib/sidekiq/client.rb +46 -66
- data/lib/sidekiq/{util.rb → component.rb} +12 -42
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
- data/lib/sidekiq/fetch.rb +22 -19
- data/lib/sidekiq/job.rb +8 -3
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +78 -55
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +62 -54
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +22 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +85 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +54 -30
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +25 -9
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +30 -18
- data/lib/sidekiq/web.rb +8 -4
- data/lib/sidekiq/worker.rb +136 -13
- data/lib/sidekiq.rb +114 -31
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +113 -60
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +50 -67
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +19 -23
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +55 -109
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +8 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_footer.erb +1 -1
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +4 -4
- data/web/views/dashboard.erb +9 -8
- data/web/views/layout.erb +1 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +14 -10
- data/web/views/queues.erb +1 -1
- metadata +27 -12
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,55 +5,20 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
9
|
-
class
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
size = if symbolized_options[:size]
|
18
|
-
symbolized_options[:size]
|
19
|
-
elsif Sidekiq.server?
|
20
|
-
# Give ourselves plenty of connections. pool is lazy
|
21
|
-
# so we won't create them until we need them.
|
22
|
-
Sidekiq.options[:concurrency] + 5
|
23
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
24
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
25
|
-
else
|
26
|
-
5
|
27
|
-
end
|
28
|
-
|
29
|
-
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
30
|
-
|
31
|
-
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
-
log_info(symbolized_options)
|
33
|
-
|
34
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
build_client(symbolized_options)
|
36
|
-
end
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
37
16
|
end
|
38
17
|
|
39
|
-
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
40
20
|
|
41
|
-
|
42
|
-
#
|
43
|
-
# We need a connection for each Processor.
|
44
|
-
# We need a connection for Pro's real-time change listener
|
45
|
-
# We need a connection to various features to call Redis every few seconds:
|
46
|
-
# - the process heartbeat.
|
47
|
-
# - enterprise's leader election
|
48
|
-
# - enterprise's cron support
|
49
|
-
def verify_sizing(size, concurrency)
|
50
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
57
22
|
if namespace
|
58
23
|
begin
|
59
24
|
require "redis/namespace"
|
@@ -68,6 +33,8 @@ module Sidekiq
|
|
68
33
|
end
|
69
34
|
end
|
70
35
|
|
36
|
+
private
|
37
|
+
|
71
38
|
def client_opts(options)
|
72
39
|
opts = options.dup
|
73
40
|
if opts[:namespace]
|
@@ -90,16 +57,80 @@ module Sidekiq
|
|
90
57
|
|
91
58
|
opts
|
92
59
|
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@adapter = RedisAdapter
|
63
|
+
|
64
|
+
class << self
|
65
|
+
attr_reader :adapter
|
66
|
+
|
67
|
+
# RedisConnection.adapter = :redis
|
68
|
+
# RedisConnection.adapter = :redis_client
|
69
|
+
def adapter=(adapter)
|
70
|
+
raise "no" if adapter == self
|
71
|
+
result = case adapter
|
72
|
+
when :redis
|
73
|
+
RedisAdapter
|
74
|
+
when Class
|
75
|
+
adapter
|
76
|
+
else
|
77
|
+
require "sidekiq/#{adapter}_adapter"
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
@adapter = result if result
|
81
|
+
end
|
82
|
+
|
83
|
+
def create(options = {})
|
84
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
85
|
+
|
86
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
87
|
+
symbolized_options[:url] = u
|
88
|
+
end
|
89
|
+
|
90
|
+
size = if symbolized_options[:size]
|
91
|
+
symbolized_options[:size]
|
92
|
+
elsif Sidekiq.server?
|
93
|
+
# Give ourselves plenty of connections. pool is lazy
|
94
|
+
# so we won't create them until we need them.
|
95
|
+
Sidekiq[:concurrency] + 5
|
96
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
97
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
98
|
+
else
|
99
|
+
5
|
100
|
+
end
|
101
|
+
|
102
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
103
|
+
|
104
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
105
|
+
log_info(symbolized_options)
|
106
|
+
|
107
|
+
redis_config = adapter.new(symbolized_options)
|
108
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
109
|
+
redis_config.new_client
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
# Sidekiq needs many concurrent Redis connections.
|
116
|
+
#
|
117
|
+
# We need a connection for each Processor.
|
118
|
+
# We need a connection for Pro's real-time change listener
|
119
|
+
# We need a connection to various features to call Redis every few seconds:
|
120
|
+
# - the process heartbeat.
|
121
|
+
# - enterprise's leader election
|
122
|
+
# - enterprise's cron support
|
123
|
+
def verify_sizing(size, concurrency)
|
124
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
125
|
+
end
|
93
126
|
|
94
127
|
def log_info(options)
|
95
128
|
redacted = "REDACTED"
|
96
129
|
|
97
|
-
#
|
98
|
-
#
|
99
|
-
#
|
100
|
-
|
101
|
-
keys = options.keys
|
102
|
-
keys.delete(:ssl_params)
|
130
|
+
# Deep clone so we can muck with these options all we want and exclude
|
131
|
+
# params from dump-and-load that may contain objects that Marshal is
|
132
|
+
# unable to safely dump.
|
133
|
+
keys = options.keys - [:logger, :ssl_params]
|
103
134
|
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
104
135
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
105
136
|
uri.password = redacted
|
@@ -112,9 +143,9 @@ module Sidekiq
|
|
112
143
|
sentinel[:password] = redacted if sentinel[:password]
|
113
144
|
end
|
114
145
|
if Sidekiq.server?
|
115
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
146
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
116
147
|
else
|
117
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
148
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
118
149
|
end
|
119
150
|
end
|
120
151
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,37 +1,63 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/
|
5
|
-
require "sidekiq/api"
|
4
|
+
require "sidekiq/component"
|
6
5
|
|
7
6
|
module Sidekiq
|
8
7
|
module Scheduled
|
9
8
|
SETS = %w[retry schedule]
|
10
9
|
|
11
10
|
class Enq
|
12
|
-
|
11
|
+
LUA_ZPOPBYSCORE = <<~LUA
|
12
|
+
local key, now = KEYS[1], ARGV[1]
|
13
|
+
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
14
|
+
if jobs[1] then
|
15
|
+
redis.call("zrem", key, jobs[1])
|
16
|
+
return jobs[1]
|
17
|
+
end
|
18
|
+
LUA
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@done = false
|
22
|
+
@lua_zpopbyscore_sha = nil
|
23
|
+
end
|
24
|
+
|
25
|
+
def enqueue_jobs(sorted_sets = SETS)
|
13
26
|
# A job's "score" in Redis is the time at which it should be processed.
|
14
27
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
28
|
Sidekiq.redis do |conn|
|
16
29
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get next
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# the queue, it's because another process already popped it so we can move on to the
|
25
|
-
# next one.
|
26
|
-
if conn.zrem(sorted_set, job)
|
27
|
-
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
-
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
-
end
|
30
|
-
end
|
30
|
+
# Get next item in the queue with score (time to execute) <= now.
|
31
|
+
# We need to go through the list one at a time to reduce the risk of something
|
32
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
33
|
+
# they are pushed onto a work queue and losing the jobs.
|
34
|
+
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
|
35
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
36
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
31
37
|
end
|
32
38
|
end
|
33
39
|
end
|
34
40
|
end
|
41
|
+
|
42
|
+
def terminate
|
43
|
+
@done = true
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def zpopbyscore(conn, keys: nil, argv: nil)
|
49
|
+
if @lua_zpopbyscore_sha.nil?
|
50
|
+
raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
|
51
|
+
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
52
|
+
end
|
53
|
+
|
54
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
|
55
|
+
rescue RedisConnection.adapter::CommandError => e
|
56
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
57
|
+
|
58
|
+
@lua_zpopbyscore_sha = nil
|
59
|
+
retry
|
60
|
+
end
|
35
61
|
end
|
36
62
|
|
37
63
|
##
|
@@ -40,12 +66,13 @@ module Sidekiq
|
|
40
66
|
# just pops the job back onto its original queue so the
|
41
67
|
# workers can pick it up like any other job.
|
42
68
|
class Poller
|
43
|
-
include
|
69
|
+
include Sidekiq::Component
|
44
70
|
|
45
71
|
INITIAL_WAIT = 10
|
46
72
|
|
47
|
-
def initialize
|
48
|
-
@
|
73
|
+
def initialize(options)
|
74
|
+
@config = options
|
75
|
+
@enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
|
49
76
|
@sleeper = ConnectionPool::TimedStack.new
|
50
77
|
@done = false
|
51
78
|
@thread = nil
|
@@ -55,6 +82,8 @@ module Sidekiq
|
|
55
82
|
# Shut down this instance, will pause until the thread is dead.
|
56
83
|
def terminate
|
57
84
|
@done = true
|
85
|
+
@enq.terminate if @enq.respond_to?(:terminate)
|
86
|
+
|
58
87
|
if @thread
|
59
88
|
t = @thread
|
60
89
|
@thread = nil
|
@@ -71,7 +100,7 @@ module Sidekiq
|
|
71
100
|
enqueue
|
72
101
|
wait
|
73
102
|
end
|
74
|
-
|
103
|
+
logger.info("Scheduler exiting...")
|
75
104
|
}
|
76
105
|
end
|
77
106
|
|
@@ -142,24 +171,19 @@ module Sidekiq
|
|
142
171
|
#
|
143
172
|
# We only do this if poll_interval_average is unset (the default).
|
144
173
|
def poll_interval_average
|
145
|
-
|
174
|
+
@config[:poll_interval_average] ||= scaled_poll_interval
|
146
175
|
end
|
147
176
|
|
148
177
|
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
149
178
|
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
150
179
|
# Redis if you run many Sidekiq processes.
|
151
180
|
def scaled_poll_interval
|
152
|
-
process_count *
|
181
|
+
process_count * @config[:average_scheduled_poll_interval]
|
153
182
|
end
|
154
183
|
|
155
184
|
def process_count
|
156
|
-
|
157
|
-
# expensive at scale. Cut it down by 90% with this counter.
|
158
|
-
# NB: This method is only called by the scheduler thread so we
|
159
|
-
# don't need to worry about the thread safety of +=.
|
160
|
-
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
185
|
+
pcount = Sidekiq.redis { |conn| conn.scard("processes") }
|
161
186
|
pcount = 1 if pcount == 0
|
162
|
-
@count_calls += 1
|
163
187
|
pcount
|
164
188
|
end
|
165
189
|
|
@@ -168,7 +192,7 @@ module Sidekiq
|
|
168
192
|
# to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
|
169
193
|
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
170
194
|
total = 0
|
171
|
-
total += INITIAL_WAIT unless
|
195
|
+
total += INITIAL_WAIT unless @config[:poll_interval_average]
|
172
196
|
total += (5 * rand)
|
173
197
|
|
174
198
|
@sleeper.pop(total)
|
@@ -4,7 +4,7 @@ require "sidekiq/testing"
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# The Sidekiq inline infrastructure overrides perform_async so that it
|
7
|
-
# actually calls perform instead. This allows
|
7
|
+
# actually calls perform instead. This allows jobs to be run inline in a
|
8
8
|
# testing environment.
|
9
9
|
#
|
10
10
|
# This is similar to `Resque.inline = true` functionality.
|
@@ -15,8 +15,8 @@ require "sidekiq/testing"
|
|
15
15
|
#
|
16
16
|
# $external_variable = 0
|
17
17
|
#
|
18
|
-
# class
|
19
|
-
# include Sidekiq::
|
18
|
+
# class ExternalJob
|
19
|
+
# include Sidekiq::Job
|
20
20
|
#
|
21
21
|
# def perform
|
22
22
|
# $external_variable = 1
|
@@ -24,7 +24,7 @@ require "sidekiq/testing"
|
|
24
24
|
# end
|
25
25
|
#
|
26
26
|
# assert_equal 0, $external_variable
|
27
|
-
#
|
27
|
+
# ExternalJob.perform_async
|
28
28
|
# assert_equal 1, $external_variable
|
29
29
|
#
|
30
30
|
Sidekiq::Testing.inline!
|