sidekiq 6.0.0 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +290 -2
- data/LICENSE +3 -3
- data/README.md +7 -9
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +222 -145
- data/lib/sidekiq/cli.rb +67 -28
- data/lib/sidekiq/client.rb +17 -34
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +5 -4
- data/lib/sidekiq/extensions/active_record.rb +6 -5
- data/lib/sidekiq/extensions/class_methods.rb +7 -6
- data/lib/sidekiq/extensions/generic_proxy.rb +5 -3
- data/lib/sidekiq/fetch.rb +36 -27
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +13 -5
- data/lib/sidekiq/job_retry.rb +33 -21
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +110 -28
- data/lib/sidekiq/logger.rb +109 -12
- data/lib/sidekiq/manager.rb +10 -12
- data/lib/sidekiq/middleware/chain.rb +17 -6
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/monitor.rb +3 -18
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +22 -24
- data/lib/sidekiq/rails.rb +27 -18
- data/lib/sidekiq/redis_connection.rb +19 -13
- data/lib/sidekiq/scheduled.rb +48 -12
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +14 -4
- data/lib/sidekiq/util.rb +40 -1
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +41 -31
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +51 -33
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +37 -73
- data/lib/sidekiq/worker.rb +133 -16
- data/lib/sidekiq.rb +29 -8
- data/sidekiq.gemspec +13 -6
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +53 -53
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +43 -232
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +6 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +5 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +19 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +34 -54
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -47
@@ -67,7 +67,6 @@ module Sidekiq
|
|
67
67
|
module Middleware
|
68
68
|
class Chain
|
69
69
|
include Enumerable
|
70
|
-
attr_reader :entries
|
71
70
|
|
72
71
|
def initialize_copy(copy)
|
73
72
|
copy.instance_variable_set(:@entries, entries.dup)
|
@@ -78,21 +77,25 @@ module Sidekiq
|
|
78
77
|
end
|
79
78
|
|
80
79
|
def initialize
|
81
|
-
@entries =
|
80
|
+
@entries = nil
|
82
81
|
yield self if block_given?
|
83
82
|
end
|
84
83
|
|
84
|
+
def entries
|
85
|
+
@entries ||= []
|
86
|
+
end
|
87
|
+
|
85
88
|
def remove(klass)
|
86
89
|
entries.delete_if { |entry| entry.klass == klass }
|
87
90
|
end
|
88
91
|
|
89
92
|
def add(klass, *args)
|
90
|
-
remove(klass)
|
93
|
+
remove(klass)
|
91
94
|
entries << Entry.new(klass, *args)
|
92
95
|
end
|
93
96
|
|
94
97
|
def prepend(klass, *args)
|
95
|
-
remove(klass)
|
98
|
+
remove(klass)
|
96
99
|
entries.insert(0, Entry.new(klass, *args))
|
97
100
|
end
|
98
101
|
|
@@ -114,6 +117,10 @@ module Sidekiq
|
|
114
117
|
any? { |entry| entry.klass == klass }
|
115
118
|
end
|
116
119
|
|
120
|
+
def empty?
|
121
|
+
@entries.nil? || @entries.empty?
|
122
|
+
end
|
123
|
+
|
117
124
|
def retrieve
|
118
125
|
map(&:make_new)
|
119
126
|
end
|
@@ -123,8 +130,10 @@ module Sidekiq
|
|
123
130
|
end
|
124
131
|
|
125
132
|
def invoke(*args)
|
126
|
-
|
127
|
-
|
133
|
+
return yield if empty?
|
134
|
+
|
135
|
+
chain = retrieve
|
136
|
+
traverse_chain = proc do
|
128
137
|
if chain.empty?
|
129
138
|
yield
|
130
139
|
else
|
@@ -135,6 +144,8 @@ module Sidekiq
|
|
135
144
|
end
|
136
145
|
end
|
137
146
|
|
147
|
+
private
|
148
|
+
|
138
149
|
class Entry
|
139
150
|
attr_reader :klass
|
140
151
|
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "active_support/current_attributes"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
##
|
5
|
+
# Automatically save and load any current attributes in the execution context
|
6
|
+
# so context attributes "flow" from Rails actions into any associated jobs.
|
7
|
+
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
8
|
+
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
9
|
+
#
|
10
|
+
# @example
|
11
|
+
#
|
12
|
+
# # in your initializer
|
13
|
+
# require "sidekiq/middleware/current_attributes"
|
14
|
+
# Sidekiq::CurrentAttributes.persist(Myapp::Current)
|
15
|
+
#
|
16
|
+
module CurrentAttributes
|
17
|
+
class Save
|
18
|
+
def initialize(cattr)
|
19
|
+
@klass = cattr
|
20
|
+
end
|
21
|
+
|
22
|
+
def call(_, job, _, _)
|
23
|
+
attrs = @klass.attributes
|
24
|
+
if job.has_key?("cattr")
|
25
|
+
job["cattr"].merge!(attrs)
|
26
|
+
else
|
27
|
+
job["cattr"] = attrs
|
28
|
+
end
|
29
|
+
yield
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class Load
|
34
|
+
def initialize(cattr)
|
35
|
+
@klass = cattr
|
36
|
+
end
|
37
|
+
|
38
|
+
def call(_, job, _, &block)
|
39
|
+
if job.has_key?("cattr")
|
40
|
+
@klass.set(job["cattr"], &block)
|
41
|
+
else
|
42
|
+
yield
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.persist(klass)
|
48
|
+
Sidekiq.configure_client do |config|
|
49
|
+
config.client_middleware.add Save, klass
|
50
|
+
end
|
51
|
+
Sidekiq.configure_server do |config|
|
52
|
+
config.client_middleware.add Save, klass
|
53
|
+
config.server_middleware.add Load, klass
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -4,21 +4,6 @@ require "fileutils"
|
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
6
|
class Sidekiq::Monitor
|
7
|
-
CMD = File.basename($PROGRAM_NAME)
|
8
|
-
|
9
|
-
attr_reader :stage
|
10
|
-
|
11
|
-
def self.print_usage
|
12
|
-
puts "#{CMD} - monitor Sidekiq from the command line."
|
13
|
-
puts
|
14
|
-
puts "Usage: #{CMD} status <section>"
|
15
|
-
puts
|
16
|
-
puts " <section> (optional) view a specific section of the status output"
|
17
|
-
puts " Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
|
18
|
-
puts
|
19
|
-
puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
|
20
|
-
end
|
21
|
-
|
22
7
|
class Status
|
23
8
|
VALID_SECTIONS = %w[all version overview processes queues]
|
24
9
|
COL_PAD = 2
|
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
|
|
47
32
|
|
48
33
|
def version
|
49
34
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
50
|
-
puts Time.now
|
35
|
+
puts Time.now.utc
|
51
36
|
end
|
52
37
|
|
53
38
|
def overview
|
@@ -77,7 +62,7 @@ class Sidekiq::Monitor
|
|
77
62
|
columns = {
|
78
63
|
name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
79
64
|
size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
80
|
-
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
65
|
+
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
81
66
|
}
|
82
67
|
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
83
68
|
puts
|
@@ -116,7 +101,7 @@ class Sidekiq::Monitor
|
|
116
101
|
tags = [
|
117
102
|
process["tag"],
|
118
103
|
process["labels"],
|
119
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
104
|
+
(process["quiet"] == "true" ? "quiet" : nil)
|
120
105
|
].flatten.compact
|
121
106
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
122
107
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -12,10 +12,10 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
Sidekiq.redis do |conn|
|
14
14
|
type = conn.type(key)
|
15
|
+
rev = opts && opts[:reverse]
|
15
16
|
|
16
17
|
case type
|
17
18
|
when "zset"
|
18
|
-
rev = opts && opts[:reverse]
|
19
19
|
total_size, items = conn.multi {
|
20
20
|
conn.zcard(key)
|
21
21
|
if rev
|
@@ -28,8 +28,13 @@ module Sidekiq
|
|
28
28
|
when "list"
|
29
29
|
total_size, items = conn.multi {
|
30
30
|
conn.llen(key)
|
31
|
-
|
31
|
+
if rev
|
32
|
+
conn.lrange(key, -ending - 1, -starting - 1)
|
33
|
+
else
|
34
|
+
conn.lrange(key, starting, ending)
|
35
|
+
end
|
32
36
|
}
|
37
|
+
items.reverse! if rev
|
33
38
|
[current_page, total_size, items]
|
34
39
|
when "none"
|
35
40
|
[1, 0, []]
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -28,15 +28,15 @@ module Sidekiq
|
|
28
28
|
attr_reader :thread
|
29
29
|
attr_reader :job
|
30
30
|
|
31
|
-
def initialize(mgr)
|
31
|
+
def initialize(mgr, options)
|
32
32
|
@mgr = mgr
|
33
33
|
@down = false
|
34
34
|
@done = false
|
35
35
|
@job = nil
|
36
36
|
@thread = nil
|
37
|
-
@strategy =
|
38
|
-
@reloader =
|
39
|
-
@job_logger = (
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
40
|
@retrier = Sidekiq::JobRetry.new
|
41
41
|
end
|
42
42
|
|
@@ -111,16 +111,19 @@ module Sidekiq
|
|
111
111
|
nil
|
112
112
|
end
|
113
113
|
|
114
|
-
def dispatch(job_hash, queue)
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
115
|
# since middleware can mutate the job hash
|
116
|
-
# we clone
|
116
|
+
# we need to clone it to report the original
|
117
117
|
# job structure to the Web UI
|
118
|
-
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
119
122
|
|
120
|
-
@job_logger.
|
121
|
-
@retrier.global(
|
123
|
+
@job_logger.prepare(job_hash) do
|
124
|
+
@retrier.global(jobstr, queue) do
|
122
125
|
@job_logger.call(job_hash, queue) do
|
123
|
-
stats(
|
126
|
+
stats(jobstr, queue) do
|
124
127
|
# Rails 5 requires a Reloader to wrap code execution. In order to
|
125
128
|
# constantize the worker and instantiate an instance, we have to call
|
126
129
|
# the Reloader. It handles code loading, db connection management, etc.
|
@@ -129,7 +132,7 @@ module Sidekiq
|
|
129
132
|
klass = constantize(job_hash["class"])
|
130
133
|
worker = klass.new
|
131
134
|
worker.jid = job_hash["jid"]
|
132
|
-
@retrier.local(worker,
|
135
|
+
@retrier.local(worker, jobstr, queue) do
|
133
136
|
yield worker
|
134
137
|
end
|
135
138
|
end
|
@@ -156,9 +159,9 @@ module Sidekiq
|
|
156
159
|
|
157
160
|
ack = false
|
158
161
|
begin
|
159
|
-
dispatch(job_hash, queue) do |worker|
|
162
|
+
dispatch(job_hash, queue, jobstr) do |worker|
|
160
163
|
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
161
|
-
execute_job(worker,
|
164
|
+
execute_job(worker, job_hash["args"])
|
162
165
|
end
|
163
166
|
end
|
164
167
|
ack = true
|
@@ -178,7 +181,7 @@ module Sidekiq
|
|
178
181
|
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
179
182
|
# so it can be rescued when using Sidekiq Pro.
|
180
183
|
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
181
|
-
raise
|
184
|
+
raise ex
|
182
185
|
ensure
|
183
186
|
if ack
|
184
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
@@ -247,8 +250,8 @@ module Sidekiq
|
|
247
250
|
FAILURE = Counter.new
|
248
251
|
WORKER_STATE = SharedWorkerState.new
|
249
252
|
|
250
|
-
def stats(
|
251
|
-
WORKER_STATE.set(tid, {queue: queue, payload:
|
253
|
+
def stats(jobstr, queue)
|
254
|
+
WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
252
255
|
|
253
256
|
begin
|
254
257
|
yield
|
@@ -261,21 +264,16 @@ module Sidekiq
|
|
261
264
|
end
|
262
265
|
end
|
263
266
|
|
264
|
-
# Deep clone the arguments passed to the worker so that if
|
265
|
-
# the job fails, what is pushed back onto Redis hasn't
|
266
|
-
# been mutated by the worker.
|
267
|
-
def cloned(thing)
|
268
|
-
Marshal.load(Marshal.dump(thing))
|
269
|
-
end
|
270
|
-
|
271
267
|
def constantize(str)
|
268
|
+
return Object.const_get(str) unless str.include?("::")
|
269
|
+
|
272
270
|
names = str.split("::")
|
273
271
|
names.shift if names.empty? || names.first.empty?
|
274
272
|
|
275
273
|
names.inject(Object) do |constant, name|
|
276
274
|
# the false flag limits search for name to under the constant namespace
|
277
275
|
# which mimics Rails' behaviour
|
278
|
-
constant.
|
276
|
+
constant.const_get(name, false)
|
279
277
|
end
|
280
278
|
end
|
281
279
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -4,6 +4,22 @@ require "sidekiq/worker"
|
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -21,10 +37,19 @@ module Sidekiq
|
|
21
37
|
end
|
22
38
|
end
|
23
39
|
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |_|
|
42
|
+
# This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
24
51
|
# This hook happens after all initializers are run, just before returning
|
25
52
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
53
|
#
|
29
54
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
55
|
config.after_initialize do
|
@@ -32,21 +57,5 @@ module Sidekiq
|
|
32
57
|
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
33
58
|
end
|
34
59
|
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
|
-
end
|
50
|
-
end
|
51
60
|
end
|
52
61
|
end
|
@@ -8,15 +8,14 @@ module Sidekiq
|
|
8
8
|
class RedisConnection
|
9
9
|
class << self
|
10
10
|
def create(options = {})
|
11
|
-
options.
|
12
|
-
options[key.to_sym] = options.delete(key)
|
13
|
-
end
|
11
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
14
12
|
|
15
|
-
|
16
|
-
|
13
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
14
|
+
symbolized_options[:url] = u
|
15
|
+
end
|
17
16
|
|
18
|
-
size = if
|
19
|
-
|
17
|
+
size = if symbolized_options[:size]
|
18
|
+
symbolized_options[:size]
|
20
19
|
elsif Sidekiq.server?
|
21
20
|
# Give ourselves plenty of connections. pool is lazy
|
22
21
|
# so we won't create them until we need them.
|
@@ -29,11 +28,11 @@ module Sidekiq
|
|
29
28
|
|
30
29
|
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
31
30
|
|
32
|
-
pool_timeout =
|
33
|
-
log_info(
|
31
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
+
log_info(symbolized_options)
|
34
33
|
|
35
34
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
36
|
-
build_client(
|
35
|
+
build_client(symbolized_options)
|
37
36
|
end
|
38
37
|
end
|
39
38
|
|
@@ -93,9 +92,13 @@ module Sidekiq
|
|
93
92
|
end
|
94
93
|
|
95
94
|
def log_info(options)
|
96
|
-
# Don't log Redis AUTH password
|
97
95
|
redacted = "REDACTED"
|
98
|
-
|
96
|
+
|
97
|
+
# Deep clone so we can muck with these options all we want and exclude
|
98
|
+
# params from dump-and-load that may contain objects that Marshal is
|
99
|
+
# unable to safely dump.
|
100
|
+
keys = options.keys - [:logger, :ssl_params]
|
101
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
99
102
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
100
103
|
uri.password = redacted
|
101
104
|
scrubbed_options[:url] = uri.to_s
|
@@ -103,6 +106,9 @@ module Sidekiq
|
|
103
106
|
if scrubbed_options[:password]
|
104
107
|
scrubbed_options[:password] = redacted
|
105
108
|
end
|
109
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
110
|
+
sentinel[:password] = redacted if sentinel[:password]
|
111
|
+
end
|
106
112
|
if Sidekiq.server?
|
107
113
|
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
108
114
|
else
|
@@ -119,7 +125,7 @@ module Sidekiq
|
|
119
125
|
# initialization code at all.
|
120
126
|
#
|
121
127
|
p = ENV["REDIS_PROVIDER"]
|
122
|
-
if p && p =~
|
128
|
+
if p && p =~ /:/
|
123
129
|
raise <<~EOM
|
124
130
|
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
125
131
|
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -9,28 +9,56 @@ module Sidekiq
|
|
9
9
|
SETS = %w[retry schedule]
|
10
10
|
|
11
11
|
class Enq
|
12
|
-
|
12
|
+
LUA_ZPOPBYSCORE = <<~LUA
|
13
|
+
local key, now = KEYS[1], ARGV[1]
|
14
|
+
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
15
|
+
if jobs[1] then
|
16
|
+
redis.call("zrem", key, jobs[1])
|
17
|
+
return jobs[1]
|
18
|
+
end
|
19
|
+
LUA
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@done = false
|
23
|
+
@lua_zpopbyscore_sha = nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def enqueue_jobs(sorted_sets = SETS)
|
13
27
|
# A job's "score" in Redis is the time at which it should be processed.
|
14
28
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
29
|
Sidekiq.redis do |conn|
|
16
30
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get
|
31
|
+
# Get next item in the queue with score (time to execute) <= now.
|
18
32
|
# We need to go through the list one at a time to reduce the risk of something
|
19
33
|
# going wrong between the time jobs are popped from the scheduled queue and when
|
20
34
|
# they are pushed onto a work queue and losing the jobs.
|
21
|
-
while (job = conn
|
22
|
-
|
23
|
-
|
24
|
-
# the queue, it's because another process already popped it so we can move on to the
|
25
|
-
# next one.
|
26
|
-
if conn.zrem(sorted_set, job)
|
27
|
-
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
-
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
-
end
|
35
|
+
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
|
36
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
37
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
30
38
|
end
|
31
39
|
end
|
32
40
|
end
|
33
41
|
end
|
42
|
+
|
43
|
+
def terminate
|
44
|
+
@done = true
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def zpopbyscore(conn, keys: nil, argv: nil)
|
50
|
+
if @lua_zpopbyscore_sha.nil?
|
51
|
+
raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
|
52
|
+
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
53
|
+
end
|
54
|
+
|
55
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
|
56
|
+
rescue Redis::CommandError => e
|
57
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
58
|
+
|
59
|
+
@lua_zpopbyscore_sha = nil
|
60
|
+
retry
|
61
|
+
end
|
34
62
|
end
|
35
63
|
|
36
64
|
##
|
@@ -48,11 +76,14 @@ module Sidekiq
|
|
48
76
|
@sleeper = ConnectionPool::TimedStack.new
|
49
77
|
@done = false
|
50
78
|
@thread = nil
|
79
|
+
@count_calls = 0
|
51
80
|
end
|
52
81
|
|
53
82
|
# Shut down this instance, will pause until the thread is dead.
|
54
83
|
def terminate
|
55
84
|
@done = true
|
85
|
+
@enq.terminate if @enq.respond_to?(:terminate)
|
86
|
+
|
56
87
|
if @thread
|
57
88
|
t = @thread
|
58
89
|
@thread = nil
|
@@ -151,8 +182,13 @@ module Sidekiq
|
|
151
182
|
end
|
152
183
|
|
153
184
|
def process_count
|
154
|
-
|
185
|
+
# The work buried within Sidekiq::ProcessSet#cleanup can be
|
186
|
+
# expensive at scale. Cut it down by 90% with this counter.
|
187
|
+
# NB: This method is only called by the scheduler thread so we
|
188
|
+
# don't need to worry about the thread safety of +=.
|
189
|
+
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
155
190
|
pcount = 1 if pcount == 0
|
191
|
+
@count_calls += 1
|
156
192
|
pcount
|
157
193
|
end
|
158
194
|
|