sidekiq 6.0.0 → 6.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +20 -0
- data/.github/workflows/ci.yml +41 -0
- data/6.0-Upgrade.md +3 -1
- data/Changes.md +163 -1
- data/Ent-Changes.md +33 -2
- data/Gemfile +2 -2
- data/Gemfile.lock +109 -113
- data/Pro-Changes.md +39 -2
- data/README.md +4 -6
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +11 -1
- data/lib/sidekiq/api.rb +130 -94
- data/lib/sidekiq/cli.rb +40 -24
- data/lib/sidekiq/client.rb +33 -12
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/fetch.rb +26 -26
- data/lib/sidekiq/job_logger.rb +12 -4
- data/lib/sidekiq/job_retry.rb +23 -10
- data/lib/sidekiq/launcher.rb +35 -10
- data/lib/sidekiq/logger.rb +108 -12
- data/lib/sidekiq/manager.rb +4 -4
- data/lib/sidekiq/middleware/chain.rb +12 -3
- data/lib/sidekiq/monitor.rb +3 -18
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +22 -24
- data/lib/sidekiq/rails.rb +16 -18
- data/lib/sidekiq/redis_connection.rb +21 -13
- data/lib/sidekiq/scheduled.rb +13 -12
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +13 -1
- data/lib/sidekiq/util.rb +0 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +23 -24
- data/lib/sidekiq/web/csrf_protection.rb +158 -0
- data/lib/sidekiq/web/helpers.rb +25 -16
- data/lib/sidekiq/web/router.rb +2 -4
- data/lib/sidekiq/web.rb +16 -8
- data/lib/sidekiq/worker.rb +8 -11
- data/lib/sidekiq.rb +22 -8
- data/sidekiq.gemspec +3 -4
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/javascripts/dashboard.js +2 -2
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application.css +16 -6
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +2 -0
- data/web/locales/fr.yml +2 -2
- data/web/locales/ja.yml +2 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +6 -3
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +5 -2
- data/web/views/queue.erb +10 -1
- data/web/views/queues.erb +9 -1
- data/web/views/retries.erb +5 -2
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +5 -2
- metadata +21 -29
- data/.circleci/config.yml +0 -61
- data/.github/issue_template.md +0 -11
data/lib/sidekiq/logger.rb
CHANGED
@@ -4,22 +4,109 @@ require "logger"
|
|
4
4
|
require "time"
|
5
5
|
|
6
6
|
module Sidekiq
|
7
|
-
|
8
|
-
def
|
9
|
-
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
current.merge!(hash)
|
10
|
+
yield
|
11
|
+
ensure
|
12
|
+
hash.each_key { |key| current.delete(key) }
|
13
|
+
end
|
10
14
|
|
11
|
-
|
15
|
+
def self.current
|
16
|
+
Thread.current[:sidekiq_context] ||= {}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module LoggingUtils
|
21
|
+
LEVELS = {
|
22
|
+
"debug" => 0,
|
23
|
+
"info" => 1,
|
24
|
+
"warn" => 2,
|
25
|
+
"error" => 3,
|
26
|
+
"fatal" => 4
|
27
|
+
}
|
28
|
+
LEVELS.default_proc = proc do |_, level|
|
29
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def debug?
|
34
|
+
level <= 0
|
35
|
+
end
|
36
|
+
|
37
|
+
def info?
|
38
|
+
level <= 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def warn?
|
42
|
+
level <= 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def error?
|
46
|
+
level <= 3
|
47
|
+
end
|
48
|
+
|
49
|
+
def fatal?
|
50
|
+
level <= 4
|
12
51
|
end
|
13
52
|
|
14
|
-
def
|
15
|
-
|
53
|
+
def local_level
|
54
|
+
Thread.current[:sidekiq_log_level]
|
55
|
+
end
|
56
|
+
|
57
|
+
def local_level=(level)
|
58
|
+
case level
|
59
|
+
when Integer
|
60
|
+
Thread.current[:sidekiq_log_level] = level
|
61
|
+
when Symbol, String
|
62
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
63
|
+
when nil
|
64
|
+
Thread.current[:sidekiq_log_level] = nil
|
65
|
+
else
|
66
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def level
|
71
|
+
local_level || super
|
72
|
+
end
|
73
|
+
|
74
|
+
# Change the thread-local level for the duration of the given block.
|
75
|
+
def log_at(level)
|
76
|
+
old_local_level = local_level
|
77
|
+
self.local_level = level
|
16
78
|
yield
|
17
79
|
ensure
|
18
|
-
|
80
|
+
self.local_level = old_local_level
|
19
81
|
end
|
20
82
|
|
21
|
-
|
22
|
-
|
83
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
84
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
85
|
+
def add(severity, message = nil, progname = nil, &block)
|
86
|
+
severity ||= ::Logger::UNKNOWN
|
87
|
+
progname ||= @progname
|
88
|
+
|
89
|
+
return true if @logdev.nil? || severity < level
|
90
|
+
|
91
|
+
if message.nil?
|
92
|
+
if block_given?
|
93
|
+
message = yield
|
94
|
+
else
|
95
|
+
message = progname
|
96
|
+
progname = @progname
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
class Logger < ::Logger
|
105
|
+
include LoggingUtils
|
106
|
+
|
107
|
+
def initialize(*args, **kwargs)
|
108
|
+
super
|
109
|
+
self.formatter = Sidekiq.log_formatter
|
23
110
|
end
|
24
111
|
|
25
112
|
module Formatters
|
@@ -29,11 +116,20 @@ module Sidekiq
|
|
29
116
|
end
|
30
117
|
|
31
118
|
def ctx
|
32
|
-
|
119
|
+
Sidekiq::Context.current
|
33
120
|
end
|
34
121
|
|
35
122
|
def format_context
|
36
|
-
|
123
|
+
if ctx.any?
|
124
|
+
" " + ctx.compact.map { |k, v|
|
125
|
+
case v
|
126
|
+
when Array
|
127
|
+
"#{k}=#{v.join(",")}"
|
128
|
+
else
|
129
|
+
"#{k}=#{v}"
|
130
|
+
end
|
131
|
+
}.join(" ")
|
132
|
+
end
|
37
133
|
end
|
38
134
|
end
|
39
135
|
|
@@ -56,7 +152,7 @@ module Sidekiq
|
|
56
152
|
pid: ::Process.pid,
|
57
153
|
tid: tid,
|
58
154
|
lvl: severity,
|
59
|
-
msg: message
|
155
|
+
msg: message
|
60
156
|
}
|
61
157
|
c = ctx
|
62
158
|
hash["ctx"] = c unless c.empty?
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -35,7 +35,7 @@ module Sidekiq
|
|
35
35
|
@done = false
|
36
36
|
@workers = Set.new
|
37
37
|
@count.times do
|
38
|
-
@workers << Processor.new(self)
|
38
|
+
@workers << Processor.new(self, options)
|
39
39
|
end
|
40
40
|
@plock = Mutex.new
|
41
41
|
end
|
@@ -56,7 +56,7 @@ module Sidekiq
|
|
56
56
|
end
|
57
57
|
|
58
58
|
# hack for quicker development / testing environment #2774
|
59
|
-
PAUSE_TIME =
|
59
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
60
60
|
|
61
61
|
def stop(deadline)
|
62
62
|
quiet
|
@@ -90,7 +90,7 @@ module Sidekiq
|
|
90
90
|
@plock.synchronize do
|
91
91
|
@workers.delete(processor)
|
92
92
|
unless @done
|
93
|
-
p = Processor.new(self)
|
93
|
+
p = Processor.new(self, options)
|
94
94
|
@workers << p
|
95
95
|
p.start
|
96
96
|
end
|
@@ -123,7 +123,7 @@ module Sidekiq
|
|
123
123
|
# contract says that jobs are run AT LEAST once. Process termination
|
124
124
|
# is delayed until we're certain the jobs are back in Redis because
|
125
125
|
# it is worse to lose a job than to run it twice.
|
126
|
-
strategy =
|
126
|
+
strategy = @options[:fetch]
|
127
127
|
strategy.bulk_requeue(jobs, @options)
|
128
128
|
end
|
129
129
|
|
@@ -67,7 +67,6 @@ module Sidekiq
|
|
67
67
|
module Middleware
|
68
68
|
class Chain
|
69
69
|
include Enumerable
|
70
|
-
attr_reader :entries
|
71
70
|
|
72
71
|
def initialize_copy(copy)
|
73
72
|
copy.instance_variable_set(:@entries, entries.dup)
|
@@ -78,10 +77,14 @@ module Sidekiq
|
|
78
77
|
end
|
79
78
|
|
80
79
|
def initialize
|
81
|
-
@entries =
|
80
|
+
@entries = nil
|
82
81
|
yield self if block_given?
|
83
82
|
end
|
84
83
|
|
84
|
+
def entries
|
85
|
+
@entries ||= []
|
86
|
+
end
|
87
|
+
|
85
88
|
def remove(klass)
|
86
89
|
entries.delete_if { |entry| entry.klass == klass }
|
87
90
|
end
|
@@ -114,6 +117,10 @@ module Sidekiq
|
|
114
117
|
any? { |entry| entry.klass == klass }
|
115
118
|
end
|
116
119
|
|
120
|
+
def empty?
|
121
|
+
@entries.nil? || @entries.empty?
|
122
|
+
end
|
123
|
+
|
117
124
|
def retrieve
|
118
125
|
map(&:make_new)
|
119
126
|
end
|
@@ -123,8 +130,10 @@ module Sidekiq
|
|
123
130
|
end
|
124
131
|
|
125
132
|
def invoke(*args)
|
133
|
+
return yield if empty?
|
134
|
+
|
126
135
|
chain = retrieve.dup
|
127
|
-
traverse_chain =
|
136
|
+
traverse_chain = proc do
|
128
137
|
if chain.empty?
|
129
138
|
yield
|
130
139
|
else
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -4,21 +4,6 @@ require "fileutils"
|
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
6
|
class Sidekiq::Monitor
|
7
|
-
CMD = File.basename($PROGRAM_NAME)
|
8
|
-
|
9
|
-
attr_reader :stage
|
10
|
-
|
11
|
-
def self.print_usage
|
12
|
-
puts "#{CMD} - monitor Sidekiq from the command line."
|
13
|
-
puts
|
14
|
-
puts "Usage: #{CMD} status <section>"
|
15
|
-
puts
|
16
|
-
puts " <section> (optional) view a specific section of the status output"
|
17
|
-
puts " Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
|
18
|
-
puts
|
19
|
-
puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
|
20
|
-
end
|
21
|
-
|
22
7
|
class Status
|
23
8
|
VALID_SECTIONS = %w[all version overview processes queues]
|
24
9
|
COL_PAD = 2
|
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
|
|
47
32
|
|
48
33
|
def version
|
49
34
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
50
|
-
puts Time.now
|
35
|
+
puts Time.now.utc
|
51
36
|
end
|
52
37
|
|
53
38
|
def overview
|
@@ -77,7 +62,7 @@ class Sidekiq::Monitor
|
|
77
62
|
columns = {
|
78
63
|
name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
79
64
|
size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
80
|
-
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
65
|
+
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
81
66
|
}
|
82
67
|
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
83
68
|
puts
|
@@ -116,7 +101,7 @@ class Sidekiq::Monitor
|
|
116
101
|
tags = [
|
117
102
|
process["tag"],
|
118
103
|
process["labels"],
|
119
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
104
|
+
(process["quiet"] == "true" ? "quiet" : nil)
|
120
105
|
].flatten.compact
|
121
106
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
122
107
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -12,10 +12,10 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
Sidekiq.redis do |conn|
|
14
14
|
type = conn.type(key)
|
15
|
+
rev = opts && opts[:reverse]
|
15
16
|
|
16
17
|
case type
|
17
18
|
when "zset"
|
18
|
-
rev = opts && opts[:reverse]
|
19
19
|
total_size, items = conn.multi {
|
20
20
|
conn.zcard(key)
|
21
21
|
if rev
|
@@ -28,8 +28,13 @@ module Sidekiq
|
|
28
28
|
when "list"
|
29
29
|
total_size, items = conn.multi {
|
30
30
|
conn.llen(key)
|
31
|
-
|
31
|
+
if rev
|
32
|
+
conn.lrange(key, -ending - 1, -starting - 1)
|
33
|
+
else
|
34
|
+
conn.lrange(key, starting, ending)
|
35
|
+
end
|
32
36
|
}
|
37
|
+
items.reverse! if rev
|
33
38
|
[current_page, total_size, items]
|
34
39
|
when "none"
|
35
40
|
[1, 0, []]
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -28,15 +28,15 @@ module Sidekiq
|
|
28
28
|
attr_reader :thread
|
29
29
|
attr_reader :job
|
30
30
|
|
31
|
-
def initialize(mgr)
|
31
|
+
def initialize(mgr, options)
|
32
32
|
@mgr = mgr
|
33
33
|
@down = false
|
34
34
|
@done = false
|
35
35
|
@job = nil
|
36
36
|
@thread = nil
|
37
|
-
@strategy =
|
38
|
-
@reloader =
|
39
|
-
@job_logger = (
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
40
|
@retrier = Sidekiq::JobRetry.new
|
41
41
|
end
|
42
42
|
|
@@ -111,16 +111,19 @@ module Sidekiq
|
|
111
111
|
nil
|
112
112
|
end
|
113
113
|
|
114
|
-
def dispatch(job_hash, queue)
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
115
|
# since middleware can mutate the job hash
|
116
|
-
# we clone
|
116
|
+
# we need to clone it to report the original
|
117
117
|
# job structure to the Web UI
|
118
|
-
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
119
122
|
|
120
|
-
@job_logger.
|
121
|
-
@retrier.global(
|
123
|
+
@job_logger.prepare(job_hash) do
|
124
|
+
@retrier.global(jobstr, queue) do
|
122
125
|
@job_logger.call(job_hash, queue) do
|
123
|
-
stats(
|
126
|
+
stats(jobstr, queue) do
|
124
127
|
# Rails 5 requires a Reloader to wrap code execution. In order to
|
125
128
|
# constantize the worker and instantiate an instance, we have to call
|
126
129
|
# the Reloader. It handles code loading, db connection management, etc.
|
@@ -129,7 +132,7 @@ module Sidekiq
|
|
129
132
|
klass = constantize(job_hash["class"])
|
130
133
|
worker = klass.new
|
131
134
|
worker.jid = job_hash["jid"]
|
132
|
-
@retrier.local(worker,
|
135
|
+
@retrier.local(worker, jobstr, queue) do
|
133
136
|
yield worker
|
134
137
|
end
|
135
138
|
end
|
@@ -156,9 +159,9 @@ module Sidekiq
|
|
156
159
|
|
157
160
|
ack = false
|
158
161
|
begin
|
159
|
-
dispatch(job_hash, queue) do |worker|
|
162
|
+
dispatch(job_hash, queue, jobstr) do |worker|
|
160
163
|
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
161
|
-
execute_job(worker,
|
164
|
+
execute_job(worker, job_hash["args"])
|
162
165
|
end
|
163
166
|
end
|
164
167
|
ack = true
|
@@ -178,7 +181,7 @@ module Sidekiq
|
|
178
181
|
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
179
182
|
# so it can be rescued when using Sidekiq Pro.
|
180
183
|
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
181
|
-
raise
|
184
|
+
raise ex
|
182
185
|
ensure
|
183
186
|
if ack
|
184
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
@@ -247,8 +250,8 @@ module Sidekiq
|
|
247
250
|
FAILURE = Counter.new
|
248
251
|
WORKER_STATE = SharedWorkerState.new
|
249
252
|
|
250
|
-
def stats(
|
251
|
-
WORKER_STATE.set(tid, {queue: queue, payload:
|
253
|
+
def stats(jobstr, queue)
|
254
|
+
WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
252
255
|
|
253
256
|
begin
|
254
257
|
yield
|
@@ -261,21 +264,16 @@ module Sidekiq
|
|
261
264
|
end
|
262
265
|
end
|
263
266
|
|
264
|
-
# Deep clone the arguments passed to the worker so that if
|
265
|
-
# the job fails, what is pushed back onto Redis hasn't
|
266
|
-
# been mutated by the worker.
|
267
|
-
def cloned(thing)
|
268
|
-
Marshal.load(Marshal.dump(thing))
|
269
|
-
end
|
270
|
-
|
271
267
|
def constantize(str)
|
268
|
+
return Object.const_get(str) unless str.include?("::")
|
269
|
+
|
272
270
|
names = str.split("::")
|
273
271
|
names.shift if names.empty? || names.first.empty?
|
274
272
|
|
275
273
|
names.inject(Object) do |constant, name|
|
276
274
|
# the false flag limits search for name to under the constant namespace
|
277
275
|
# which mimics Rails' behaviour
|
278
|
-
constant.
|
276
|
+
constant.const_get(name, false)
|
279
277
|
end
|
280
278
|
end
|
281
279
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -4,6 +4,22 @@ require "sidekiq/worker"
|
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -23,8 +39,6 @@ module Sidekiq
|
|
23
39
|
|
24
40
|
# This hook happens after all initializers are run, just before returning
|
25
41
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
42
|
#
|
29
43
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
44
|
config.after_initialize do
|
@@ -32,21 +46,5 @@ module Sidekiq
|
|
32
46
|
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
33
47
|
end
|
34
48
|
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
|
-
end
|
50
|
-
end
|
51
49
|
end
|
52
50
|
end
|
@@ -8,15 +8,14 @@ module Sidekiq
|
|
8
8
|
class RedisConnection
|
9
9
|
class << self
|
10
10
|
def create(options = {})
|
11
|
-
options.
|
12
|
-
options[key.to_sym] = options.delete(key)
|
13
|
-
end
|
11
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
14
12
|
|
15
|
-
|
16
|
-
|
13
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
14
|
+
symbolized_options[:url] = u
|
15
|
+
end
|
17
16
|
|
18
|
-
size = if
|
19
|
-
|
17
|
+
size = if symbolized_options[:size]
|
18
|
+
symbolized_options[:size]
|
20
19
|
elsif Sidekiq.server?
|
21
20
|
# Give ourselves plenty of connections. pool is lazy
|
22
21
|
# so we won't create them until we need them.
|
@@ -29,11 +28,11 @@ module Sidekiq
|
|
29
28
|
|
30
29
|
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
31
30
|
|
32
|
-
pool_timeout =
|
33
|
-
log_info(
|
31
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
+
log_info(symbolized_options)
|
34
33
|
|
35
34
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
36
|
-
build_client(
|
35
|
+
build_client(symbolized_options)
|
37
36
|
end
|
38
37
|
end
|
39
38
|
|
@@ -93,9 +92,15 @@ module Sidekiq
|
|
93
92
|
end
|
94
93
|
|
95
94
|
def log_info(options)
|
96
|
-
# Don't log Redis AUTH password
|
97
95
|
redacted = "REDACTED"
|
98
|
-
|
96
|
+
|
97
|
+
# deep clone so we can muck with these options all we want
|
98
|
+
#
|
99
|
+
# exclude SSL params from dump-and-load because some information isn't
|
100
|
+
# safely dumpable in current Rubies
|
101
|
+
keys = options.keys
|
102
|
+
keys.delete(:ssl_params)
|
103
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
99
104
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
100
105
|
uri.password = redacted
|
101
106
|
scrubbed_options[:url] = uri.to_s
|
@@ -103,6 +108,9 @@ module Sidekiq
|
|
103
108
|
if scrubbed_options[:password]
|
104
109
|
scrubbed_options[:password] = redacted
|
105
110
|
end
|
111
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
112
|
+
sentinel[:password] = redacted if sentinel[:password]
|
113
|
+
end
|
106
114
|
if Sidekiq.server?
|
107
115
|
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
108
116
|
else
|
@@ -119,7 +127,7 @@ module Sidekiq
|
|
119
127
|
# initialization code at all.
|
120
128
|
#
|
121
129
|
p = ENV["REDIS_PROVIDER"]
|
122
|
-
if p && p =~
|
130
|
+
if p && p =~ /:/
|
123
131
|
raise <<~EOM
|
124
132
|
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
125
133
|
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -14,18 +14,19 @@ module Sidekiq
|
|
14
14
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
15
|
Sidekiq.redis do |conn|
|
16
16
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
17
|
+
# Get next items in the queue with scores (time to execute) <= now.
|
18
|
+
until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
|
19
|
+
# We need to go through the list one at a time to reduce the risk of something
|
20
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
21
|
+
# they are pushed onto a work queue and losing the jobs.
|
22
|
+
jobs.each do |job|
|
23
|
+
# Pop item off the queue and add it to the work queue. If the job can't be popped from
|
24
|
+
# the queue, it's because another process already popped it so we can move on to the
|
25
|
+
# next one.
|
26
|
+
if conn.zrem(sorted_set, job)
|
27
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
+
end
|
29
30
|
end
|
30
31
|
end
|
31
32
|
end
|