sidekiq 6.2.1 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +132 -1
- data/LICENSE +3 -3
- data/README.md +9 -4
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +192 -135
- data/lib/sidekiq/cli.rb +59 -40
- data/lib/sidekiq/client.rb +46 -66
- data/lib/sidekiq/{util.rb → component.rb} +11 -42
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +23 -20
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +37 -38
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +67 -65
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +27 -16
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +22 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +85 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +1 -1
- data/lib/sidekiq/web/application.rb +9 -6
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +14 -26
- data/lib/sidekiq/web.rb +6 -5
- data/lib/sidekiq/worker.rb +136 -13
- data/lib/sidekiq.rb +105 -30
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +113 -60
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +28 -45
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +24 -237
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +7 -0
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +8 -8
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +1 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +3 -3
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +17 -10
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
@@ -33,17 +33,35 @@ module Sidekiq
|
|
33
33
|
# end
|
34
34
|
initializer "sidekiq.active_job_integration" do
|
35
35
|
ActiveSupport.on_load(:active_job) do
|
36
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
40
58
|
# This hook happens after all initializers are run, just before returning
|
41
59
|
# from config/environment.rb back to sidekiq/cli.rb.
|
42
60
|
#
|
43
61
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
44
62
|
config.after_initialize do
|
45
|
-
Sidekiq.configure_server do |
|
46
|
-
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
47
65
|
end
|
48
66
|
end
|
49
67
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,55 +5,20 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
9
|
-
class
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
size = if symbolized_options[:size]
|
18
|
-
symbolized_options[:size]
|
19
|
-
elsif Sidekiq.server?
|
20
|
-
# Give ourselves plenty of connections. pool is lazy
|
21
|
-
# so we won't create them until we need them.
|
22
|
-
Sidekiq.options[:concurrency] + 5
|
23
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
24
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
25
|
-
else
|
26
|
-
5
|
27
|
-
end
|
28
|
-
|
29
|
-
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
30
|
-
|
31
|
-
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
-
log_info(symbolized_options)
|
33
|
-
|
34
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
build_client(symbolized_options)
|
36
|
-
end
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
37
16
|
end
|
38
17
|
|
39
|
-
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
40
20
|
|
41
|
-
|
42
|
-
#
|
43
|
-
# We need a connection for each Processor.
|
44
|
-
# We need a connection for Pro's real-time change listener
|
45
|
-
# We need a connection to various features to call Redis every few seconds:
|
46
|
-
# - the process heartbeat.
|
47
|
-
# - enterprise's leader election
|
48
|
-
# - enterprise's cron support
|
49
|
-
def verify_sizing(size, concurrency)
|
50
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
57
22
|
if namespace
|
58
23
|
begin
|
59
24
|
require "redis/namespace"
|
@@ -68,6 +33,8 @@ module Sidekiq
|
|
68
33
|
end
|
69
34
|
end
|
70
35
|
|
36
|
+
private
|
37
|
+
|
71
38
|
def client_opts(options)
|
72
39
|
opts = options.dup
|
73
40
|
if opts[:namespace]
|
@@ -90,16 +57,80 @@ module Sidekiq
|
|
90
57
|
|
91
58
|
opts
|
92
59
|
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@adapter = RedisAdapter
|
63
|
+
|
64
|
+
class << self
|
65
|
+
attr_reader :adapter
|
66
|
+
|
67
|
+
# RedisConnection.adapter = :redis
|
68
|
+
# RedisConnection.adapter = :redis_client
|
69
|
+
def adapter=(adapter)
|
70
|
+
raise "no" if adapter == self
|
71
|
+
result = case adapter
|
72
|
+
when :redis
|
73
|
+
RedisAdapter
|
74
|
+
when Class
|
75
|
+
adapter
|
76
|
+
else
|
77
|
+
require "sidekiq/#{adapter}_adapter"
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
@adapter = result if result
|
81
|
+
end
|
82
|
+
|
83
|
+
def create(options = {})
|
84
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
85
|
+
|
86
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
87
|
+
symbolized_options[:url] = u
|
88
|
+
end
|
89
|
+
|
90
|
+
size = if symbolized_options[:size]
|
91
|
+
symbolized_options[:size]
|
92
|
+
elsif Sidekiq.server?
|
93
|
+
# Give ourselves plenty of connections. pool is lazy
|
94
|
+
# so we won't create them until we need them.
|
95
|
+
Sidekiq[:concurrency] + 5
|
96
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
97
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
98
|
+
else
|
99
|
+
5
|
100
|
+
end
|
101
|
+
|
102
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
103
|
+
|
104
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
105
|
+
log_info(symbolized_options)
|
106
|
+
|
107
|
+
redis_config = adapter.new(symbolized_options)
|
108
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
109
|
+
redis_config.new_client
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
# Sidekiq needs many concurrent Redis connections.
|
116
|
+
#
|
117
|
+
# We need a connection for each Processor.
|
118
|
+
# We need a connection for Pro's real-time change listener
|
119
|
+
# We need a connection to various features to call Redis every few seconds:
|
120
|
+
# - the process heartbeat.
|
121
|
+
# - enterprise's leader election
|
122
|
+
# - enterprise's cron support
|
123
|
+
def verify_sizing(size, concurrency)
|
124
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
125
|
+
end
|
93
126
|
|
94
127
|
def log_info(options)
|
95
128
|
redacted = "REDACTED"
|
96
129
|
|
97
|
-
#
|
98
|
-
#
|
99
|
-
#
|
100
|
-
|
101
|
-
keys = options.keys
|
102
|
-
keys.delete(:ssl_params)
|
130
|
+
# Deep clone so we can muck with these options all we want and exclude
|
131
|
+
# params from dump-and-load that may contain objects that Marshal is
|
132
|
+
# unable to safely dump.
|
133
|
+
keys = options.keys - [:logger, :ssl_params]
|
103
134
|
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
104
135
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
105
136
|
uri.password = redacted
|
@@ -112,9 +143,9 @@ module Sidekiq
|
|
112
143
|
sentinel[:password] = redacted if sentinel[:password]
|
113
144
|
end
|
114
145
|
if Sidekiq.server?
|
115
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
146
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
116
147
|
else
|
117
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
148
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
118
149
|
end
|
119
150
|
end
|
120
151
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,37 +1,64 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/util"
|
5
4
|
require "sidekiq/api"
|
5
|
+
require "sidekiq/component"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
module Scheduled
|
9
9
|
SETS = %w[retry schedule]
|
10
10
|
|
11
11
|
class Enq
|
12
|
-
|
12
|
+
LUA_ZPOPBYSCORE = <<~LUA
|
13
|
+
local key, now = KEYS[1], ARGV[1]
|
14
|
+
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
15
|
+
if jobs[1] then
|
16
|
+
redis.call("zrem", key, jobs[1])
|
17
|
+
return jobs[1]
|
18
|
+
end
|
19
|
+
LUA
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@done = false
|
23
|
+
@lua_zpopbyscore_sha = nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def enqueue_jobs(sorted_sets = SETS)
|
13
27
|
# A job's "score" in Redis is the time at which it should be processed.
|
14
28
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
29
|
Sidekiq.redis do |conn|
|
16
30
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get next
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# the queue, it's because another process already popped it so we can move on to the
|
25
|
-
# next one.
|
26
|
-
if conn.zrem(sorted_set, job)
|
27
|
-
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
-
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
-
end
|
30
|
-
end
|
31
|
+
# Get next item in the queue with score (time to execute) <= now.
|
32
|
+
# We need to go through the list one at a time to reduce the risk of something
|
33
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
34
|
+
# they are pushed onto a work queue and losing the jobs.
|
35
|
+
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
|
36
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
37
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
31
38
|
end
|
32
39
|
end
|
33
40
|
end
|
34
41
|
end
|
42
|
+
|
43
|
+
def terminate
|
44
|
+
@done = true
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def zpopbyscore(conn, keys: nil, argv: nil)
|
50
|
+
if @lua_zpopbyscore_sha.nil?
|
51
|
+
raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
|
52
|
+
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
53
|
+
end
|
54
|
+
|
55
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
|
56
|
+
rescue RedisConnection.adapter::CommandError => e
|
57
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
58
|
+
|
59
|
+
@lua_zpopbyscore_sha = nil
|
60
|
+
retry
|
61
|
+
end
|
35
62
|
end
|
36
63
|
|
37
64
|
##
|
@@ -40,20 +67,24 @@ module Sidekiq
|
|
40
67
|
# just pops the job back onto its original queue so the
|
41
68
|
# workers can pick it up like any other job.
|
42
69
|
class Poller
|
43
|
-
include
|
70
|
+
include Sidekiq::Component
|
44
71
|
|
45
72
|
INITIAL_WAIT = 10
|
46
73
|
|
47
|
-
def initialize
|
48
|
-
@
|
74
|
+
def initialize(options)
|
75
|
+
@config = options
|
76
|
+
@enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
|
49
77
|
@sleeper = ConnectionPool::TimedStack.new
|
50
78
|
@done = false
|
51
79
|
@thread = nil
|
80
|
+
@count_calls = 0
|
52
81
|
end
|
53
82
|
|
54
83
|
# Shut down this instance, will pause until the thread is dead.
|
55
84
|
def terminate
|
56
85
|
@done = true
|
86
|
+
@enq.terminate if @enq.respond_to?(:terminate)
|
87
|
+
|
57
88
|
if @thread
|
58
89
|
t = @thread
|
59
90
|
@thread = nil
|
@@ -70,7 +101,7 @@ module Sidekiq
|
|
70
101
|
enqueue
|
71
102
|
wait
|
72
103
|
end
|
73
|
-
|
104
|
+
logger.info("Scheduler exiting...")
|
74
105
|
}
|
75
106
|
end
|
76
107
|
|
@@ -141,19 +172,24 @@ module Sidekiq
|
|
141
172
|
#
|
142
173
|
# We only do this if poll_interval_average is unset (the default).
|
143
174
|
def poll_interval_average
|
144
|
-
|
175
|
+
@config[:poll_interval_average] ||= scaled_poll_interval
|
145
176
|
end
|
146
177
|
|
147
178
|
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
148
179
|
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
149
180
|
# Redis if you run many Sidekiq processes.
|
150
181
|
def scaled_poll_interval
|
151
|
-
process_count *
|
182
|
+
process_count * @config[:average_scheduled_poll_interval]
|
152
183
|
end
|
153
184
|
|
154
185
|
def process_count
|
155
|
-
|
186
|
+
# The work buried within Sidekiq::ProcessSet#cleanup can be
|
187
|
+
# expensive at scale. Cut it down by 90% with this counter.
|
188
|
+
# NB: This method is only called by the scheduler thread so we
|
189
|
+
# don't need to worry about the thread safety of +=.
|
190
|
+
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
156
191
|
pcount = 1 if pcount == 0
|
192
|
+
@count_calls += 1
|
157
193
|
pcount
|
158
194
|
end
|
159
195
|
|
@@ -162,7 +198,7 @@ module Sidekiq
|
|
162
198
|
# to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
|
163
199
|
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
164
200
|
total = 0
|
165
|
-
total += INITIAL_WAIT unless
|
201
|
+
total += INITIAL_WAIT unless @config[:poll_interval_average]
|
166
202
|
total += (5 * rand)
|
167
203
|
|
168
204
|
@sleeper.pop(total)
|
@@ -4,7 +4,7 @@ require "sidekiq/testing"
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# The Sidekiq inline infrastructure overrides perform_async so that it
|
7
|
-
# actually calls perform instead. This allows
|
7
|
+
# actually calls perform instead. This allows jobs to be run inline in a
|
8
8
|
# testing environment.
|
9
9
|
#
|
10
10
|
# This is similar to `Resque.inline = true` functionality.
|
@@ -15,8 +15,8 @@ require "sidekiq/testing"
|
|
15
15
|
#
|
16
16
|
# $external_variable = 0
|
17
17
|
#
|
18
|
-
# class
|
19
|
-
# include Sidekiq::
|
18
|
+
# class ExternalJob
|
19
|
+
# include Sidekiq::Job
|
20
20
|
#
|
21
21
|
# def perform
|
22
22
|
# $external_variable = 1
|
@@ -24,7 +24,7 @@ require "sidekiq/testing"
|
|
24
24
|
# end
|
25
25
|
#
|
26
26
|
# assert_equal 0, $external_variable
|
27
|
-
#
|
27
|
+
# ExternalJob.perform_async
|
28
28
|
# assert_equal 1, $external_variable
|
29
29
|
#
|
30
30
|
Sidekiq::Testing.inline!
|