sidekiq 6.2.1 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +132 -1
- data/LICENSE +3 -3
- data/README.md +9 -4
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +192 -135
- data/lib/sidekiq/cli.rb +59 -40
- data/lib/sidekiq/client.rb +46 -66
- data/lib/sidekiq/{util.rb → component.rb} +11 -42
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +23 -20
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +37 -38
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +67 -65
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +27 -16
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +22 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +85 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +1 -1
- data/lib/sidekiq/web/application.rb +9 -6
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +14 -26
- data/lib/sidekiq/web.rb +6 -5
- data/lib/sidekiq/worker.rb +136 -13
- data/lib/sidekiq.rb +105 -30
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +113 -60
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +28 -45
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +24 -237
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +7 -0
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +8 -8
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +1 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +3 -3
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +17 -10
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
+
require "sidekiq/component"
|
4
5
|
|
5
|
-
module Sidekiq
|
6
|
+
module Sidekiq # :nodoc:
|
6
7
|
class BasicFetch
|
8
|
+
include Sidekiq::Component
|
7
9
|
# We want the fetch operation to timeout every few seconds so the thread
|
8
10
|
# can check if the process is shutting down.
|
9
11
|
TIMEOUT = 2
|
10
12
|
|
11
|
-
UnitOfWork = Struct.new(:queue, :job) {
|
13
|
+
UnitOfWork = Struct.new(:queue, :job, :config) {
|
12
14
|
def acknowledge
|
13
15
|
# nothing to do
|
14
16
|
end
|
@@ -18,17 +20,17 @@ module Sidekiq
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def requeue
|
21
|
-
|
23
|
+
config.redis do |conn|
|
22
24
|
conn.rpush(queue, job)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
}
|
26
28
|
|
27
|
-
def initialize(
|
28
|
-
raise ArgumentError, "missing queue list" unless
|
29
|
-
@
|
30
|
-
@strictly_ordered_queues = !!@
|
31
|
-
@queues = @
|
29
|
+
def initialize(config)
|
30
|
+
raise ArgumentError, "missing queue list" unless config[:queues]
|
31
|
+
@config = config
|
32
|
+
@strictly_ordered_queues = !!@config[:strict]
|
33
|
+
@queues = @config[:queues].map { |q| "queue:#{q}" }
|
32
34
|
if @strictly_ordered_queues
|
33
35
|
@queues.uniq!
|
34
36
|
@queues << TIMEOUT
|
@@ -40,34 +42,34 @@ module Sidekiq
|
|
40
42
|
# 4825 Sidekiq Pro with all queues paused will return an
|
41
43
|
# empty set of queues with a trailing TIMEOUT value.
|
42
44
|
if qs.size <= 1
|
43
|
-
sleep(
|
45
|
+
sleep(TIMEOUT)
|
44
46
|
return nil
|
45
47
|
end
|
46
48
|
|
47
|
-
|
48
|
-
UnitOfWork.new(
|
49
|
+
queue, job = redis { |conn| conn.brpop(*qs) }
|
50
|
+
UnitOfWork.new(queue, job, config) if queue
|
49
51
|
end
|
50
52
|
|
51
53
|
def bulk_requeue(inprogress, options)
|
52
54
|
return if inprogress.empty?
|
53
55
|
|
54
|
-
|
56
|
+
logger.debug { "Re-queueing terminated jobs" }
|
55
57
|
jobs_to_requeue = {}
|
56
58
|
inprogress.each do |unit_of_work|
|
57
59
|
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
60
|
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
59
61
|
end
|
60
62
|
|
61
|
-
|
62
|
-
conn.pipelined do
|
63
|
+
redis do |conn|
|
64
|
+
conn.pipelined do |pipeline|
|
63
65
|
jobs_to_requeue.each do |queue, jobs|
|
64
|
-
|
66
|
+
pipeline.rpush(queue, jobs)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
68
|
-
|
70
|
+
logger.info("Pushed #{inprogress.size} jobs back to Redis")
|
69
71
|
rescue => ex
|
70
|
-
|
72
|
+
logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
71
73
|
end
|
72
74
|
|
73
75
|
# Creating the Redis#brpop command takes into account any
|
@@ -79,9 +81,10 @@ module Sidekiq
|
|
79
81
|
if @strictly_ordered_queues
|
80
82
|
@queues
|
81
83
|
else
|
82
|
-
|
83
|
-
|
84
|
-
|
84
|
+
permute = @queues.shuffle
|
85
|
+
permute.uniq!
|
86
|
+
permute << TIMEOUT
|
87
|
+
permute
|
85
88
|
end
|
86
89
|
end
|
87
90
|
end
|
data/lib/sidekiq/job.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "sidekiq/worker"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
|
5
|
+
# Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
|
6
|
+
#
|
7
|
+
# The term "worker" is too generic and overly confusing, used in several
|
8
|
+
# different contexts meaning different things. Many people call a Sidekiq
|
9
|
+
# process a "worker". Some people call the thread that executes jobs a
|
10
|
+
# "worker". This change brings Sidekiq closer to ActiveJob where your job
|
11
|
+
# classes extend ApplicationJob.
|
12
|
+
Job = Worker
|
13
|
+
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -12,46 +12,34 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
yield
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
end
|
15
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
16
|
+
@logger.info("done")
|
18
17
|
rescue Exception
|
19
|
-
|
20
|
-
|
21
|
-
end
|
18
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
19
|
+
@logger.info("fail")
|
22
20
|
|
23
21
|
raise
|
24
22
|
end
|
25
23
|
|
26
24
|
def prepare(job_hash, &block)
|
27
|
-
level = job_hash["log_level"]
|
28
|
-
if level
|
29
|
-
@logger.log_at(level) do
|
30
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
-
end
|
32
|
-
else
|
33
|
-
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def job_hash_context(job_hash)
|
38
25
|
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
39
26
|
# attribute to expose the underlying thing.
|
40
27
|
h = {
|
41
|
-
class: job_hash["wrapped"] || job_hash["class"],
|
28
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
|
42
29
|
jid: job_hash["jid"]
|
43
30
|
}
|
44
|
-
h[:bid] = job_hash["bid"] if job_hash
|
45
|
-
h[:tags] = job_hash["tags"] if job_hash
|
46
|
-
h
|
47
|
-
end
|
48
|
-
|
49
|
-
def with_elapsed_time_context(start, &block)
|
50
|
-
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
51
|
-
end
|
31
|
+
h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
|
32
|
+
h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
|
52
33
|
|
53
|
-
|
54
|
-
|
34
|
+
Thread.current[:sidekiq_context] = h
|
35
|
+
level = job_hash["log_level"]
|
36
|
+
if level
|
37
|
+
@logger.log_at(level, &block)
|
38
|
+
else
|
39
|
+
yield
|
40
|
+
end
|
41
|
+
ensure
|
42
|
+
Thread.current[:sidekiq_context] = nil
|
55
43
|
end
|
56
44
|
|
57
45
|
private
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -25,18 +25,19 @@ module Sidekiq
|
|
25
25
|
#
|
26
26
|
# A job looks like:
|
27
27
|
#
|
28
|
-
# { 'class' => '
|
28
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
|
29
29
|
#
|
30
30
|
# The 'retry' option also accepts a number (in place of 'true'):
|
31
31
|
#
|
32
|
-
# { 'class' => '
|
32
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
|
33
33
|
#
|
34
34
|
# The job will be retried this number of times before giving up. (If simply
|
35
35
|
# 'true', Sidekiq retries 25 times)
|
36
36
|
#
|
37
|
-
#
|
37
|
+
# Relevant options for job retries:
|
38
38
|
#
|
39
|
-
# * 'queue' - the queue
|
39
|
+
# * 'queue' - the queue for the initial job
|
40
|
+
# * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
|
40
41
|
# * 'retry_count' - number of times we've retried so far.
|
41
42
|
# * 'error_message' - the message from the exception
|
42
43
|
# * 'error_class' - the exception class
|
@@ -52,11 +53,12 @@ module Sidekiq
|
|
52
53
|
#
|
53
54
|
# Sidekiq.options[:max_retries] = 7
|
54
55
|
#
|
55
|
-
# or limit the number of retries for a particular
|
56
|
+
# or limit the number of retries for a particular job and send retries to
|
57
|
+
# a low priority queue with:
|
56
58
|
#
|
57
|
-
# class
|
58
|
-
# include Sidekiq::
|
59
|
-
# sidekiq_options :
|
59
|
+
# class MyJob
|
60
|
+
# include Sidekiq::Job
|
61
|
+
# sidekiq_options retry: 10, retry_queue: 'low'
|
60
62
|
# end
|
61
63
|
#
|
62
64
|
class JobRetry
|
@@ -64,17 +66,18 @@ module Sidekiq
|
|
64
66
|
|
65
67
|
class Skip < Handled; end
|
66
68
|
|
67
|
-
include Sidekiq::
|
69
|
+
include Sidekiq::Component
|
68
70
|
|
69
71
|
DEFAULT_MAX_RETRY_ATTEMPTS = 25
|
70
72
|
|
71
|
-
def initialize(options
|
72
|
-
@
|
73
|
+
def initialize(options)
|
74
|
+
@config = options
|
75
|
+
@max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
|
73
76
|
end
|
74
77
|
|
75
78
|
# The global retry handler requires only the barest of data.
|
76
79
|
# We want to be able to retry as much as possible so we don't
|
77
|
-
# require the
|
80
|
+
# require the job to be instantiated.
|
78
81
|
def global(jobstr, queue)
|
79
82
|
yield
|
80
83
|
rescue Handled => ex
|
@@ -101,14 +104,14 @@ module Sidekiq
|
|
101
104
|
end
|
102
105
|
|
103
106
|
# The local retry support means that any errors that occur within
|
104
|
-
# this block can be associated with the given
|
107
|
+
# this block can be associated with the given job instance.
|
105
108
|
# This is required to support the `sidekiq_retries_exhausted` block.
|
106
109
|
#
|
107
110
|
# Note that any exception from the block is wrapped in the Skip
|
108
111
|
# exception so the global block does not reprocess the error. The
|
109
112
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
110
113
|
# calling the handle_exception handlers.
|
111
|
-
def local(
|
114
|
+
def local(jobinst, jobstr, queue)
|
112
115
|
yield
|
113
116
|
rescue Handled => ex
|
114
117
|
raise ex
|
@@ -121,11 +124,11 @@ module Sidekiq
|
|
121
124
|
|
122
125
|
msg = Sidekiq.load_json(jobstr)
|
123
126
|
if msg["retry"].nil?
|
124
|
-
msg["retry"] =
|
127
|
+
msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
|
125
128
|
end
|
126
129
|
|
127
130
|
raise e unless msg["retry"]
|
128
|
-
attempt_retry(
|
131
|
+
attempt_retry(jobinst, msg, queue, e)
|
129
132
|
# We've handled this error associated with this job, don't
|
130
133
|
# need to handle it at the global level
|
131
134
|
raise Skip
|
@@ -133,10 +136,10 @@ module Sidekiq
|
|
133
136
|
|
134
137
|
private
|
135
138
|
|
136
|
-
# Note that +
|
137
|
-
# instantiate the
|
139
|
+
# Note that +jobinst+ can be nil here if an error is raised before we can
|
140
|
+
# instantiate the job instance. All access must be guarded and
|
138
141
|
# best effort.
|
139
|
-
def attempt_retry(
|
142
|
+
def attempt_retry(jobinst, msg, queue, exception)
|
140
143
|
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
141
144
|
|
142
145
|
msg["queue"] = (msg["retry_queue"] || queue)
|
@@ -168,23 +171,23 @@ module Sidekiq
|
|
168
171
|
end
|
169
172
|
|
170
173
|
if count < max_retry_attempts
|
171
|
-
delay = delay_for(
|
174
|
+
delay = delay_for(jobinst, count, exception)
|
172
175
|
# Logging here can break retries if the logging device raises ENOSPC #3979
|
173
176
|
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
174
177
|
retry_at = Time.now.to_f + delay
|
175
178
|
payload = Sidekiq.dump_json(msg)
|
176
|
-
|
179
|
+
redis do |conn|
|
177
180
|
conn.zadd("retry", retry_at.to_s, payload)
|
178
181
|
end
|
179
182
|
else
|
180
183
|
# Goodbye dear message, you (re)tried your best I'm sure.
|
181
|
-
retries_exhausted(
|
184
|
+
retries_exhausted(jobinst, msg, exception)
|
182
185
|
end
|
183
186
|
end
|
184
187
|
|
185
|
-
def retries_exhausted(
|
188
|
+
def retries_exhausted(jobinst, msg, exception)
|
186
189
|
begin
|
187
|
-
block =
|
190
|
+
block = jobinst&.sidekiq_retries_exhausted_block
|
188
191
|
block&.call(msg, exception)
|
189
192
|
rescue => e
|
190
193
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
@@ -192,7 +195,7 @@ module Sidekiq
|
|
192
195
|
|
193
196
|
send_to_morgue(msg) unless msg["dead"] == false
|
194
197
|
|
195
|
-
|
198
|
+
config.death_handlers.each do |handler|
|
196
199
|
handler.call(msg, exception)
|
197
200
|
rescue => e
|
198
201
|
handle_exception(e, {context: "Error calling death handler", job: msg})
|
@@ -213,23 +216,19 @@ module Sidekiq
|
|
213
216
|
end
|
214
217
|
end
|
215
218
|
|
216
|
-
def delay_for(
|
217
|
-
|
218
|
-
|
219
|
-
|
219
|
+
def delay_for(jobinst, count, exception)
|
220
|
+
jitter = rand(10) * (count + 1)
|
221
|
+
if jobinst&.sidekiq_retry_in_block
|
222
|
+
custom_retry_in = retry_in(jobinst, count, exception).to_i
|
223
|
+
return custom_retry_in + jitter if custom_retry_in > 0
|
220
224
|
end
|
221
|
-
|
225
|
+
(count**4) + 15 + jitter
|
222
226
|
end
|
223
227
|
|
224
|
-
|
225
|
-
|
226
|
-
(count**4) + 15 + (rand(30) * (count + 1))
|
227
|
-
end
|
228
|
-
|
229
|
-
def retry_in(worker, count, exception)
|
230
|
-
worker.sidekiq_retry_in_block.call(count, exception)
|
228
|
+
def retry_in(jobinst, count, exception)
|
229
|
+
jobinst.sidekiq_retry_in_block.call(count, exception)
|
231
230
|
rescue Exception => e
|
232
|
-
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{
|
231
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
233
232
|
nil
|
234
233
|
end
|
235
234
|
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require "securerandom"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module JobUtil
|
6
|
+
# These functions encapsulate various job utilities.
|
7
|
+
|
8
|
+
TRANSIENT_ATTRIBUTES = %w[]
|
9
|
+
|
10
|
+
def validate(item)
|
11
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
12
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
13
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
14
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
15
|
+
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
16
|
+
end
|
17
|
+
|
18
|
+
def verify_json(item)
|
19
|
+
job_class = item["wrapped"] || item["class"]
|
20
|
+
if Sidekiq[:on_complex_arguments] == :raise
|
21
|
+
msg = <<~EOM
|
22
|
+
Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
|
23
|
+
To disable this error, remove `Sidekiq.strict_args!` from your initializer.
|
24
|
+
EOM
|
25
|
+
raise(ArgumentError, msg) unless json_safe?(item)
|
26
|
+
elsif Sidekiq[:on_complex_arguments] == :warn
|
27
|
+
Sidekiq.logger.warn <<~EOM unless json_safe?(item)
|
28
|
+
Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
|
29
|
+
Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
|
30
|
+
by calling `Sidekiq.strict_args!` during Sidekiq initialization.
|
31
|
+
EOM
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def normalize_item(item)
|
36
|
+
validate(item)
|
37
|
+
|
38
|
+
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
39
|
+
# this allows ActiveJobs to control sidekiq_options too.
|
40
|
+
defaults = normalized_hash(item["class"])
|
41
|
+
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
|
42
|
+
item = defaults.merge(item)
|
43
|
+
|
44
|
+
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
45
|
+
|
46
|
+
# remove job attributes which aren't necessary to persist into Redis
|
47
|
+
TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
|
48
|
+
|
49
|
+
item["jid"] ||= SecureRandom.hex(12)
|
50
|
+
item["class"] = item["class"].to_s
|
51
|
+
item["queue"] = item["queue"].to_s
|
52
|
+
item["created_at"] ||= Time.now.to_f
|
53
|
+
item
|
54
|
+
end
|
55
|
+
|
56
|
+
def normalized_hash(item_class)
|
57
|
+
if item_class.is_a?(Class)
|
58
|
+
raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
|
59
|
+
item_class.get_sidekiq_options
|
60
|
+
else
|
61
|
+
Sidekiq.default_job_options
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def json_safe?(item)
|
68
|
+
JSON.parse(JSON.dump(item["args"])) == item["args"]
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -3,11 +3,12 @@
|
|
3
3
|
require "sidekiq/manager"
|
4
4
|
require "sidekiq/fetch"
|
5
5
|
require "sidekiq/scheduled"
|
6
|
+
require "sidekiq/ring_buffer"
|
6
7
|
|
7
8
|
module Sidekiq
|
8
9
|
# The Launcher starts the Manager and Poller threads and provides the process heartbeat.
|
9
10
|
class Launcher
|
10
|
-
include
|
11
|
+
include Sidekiq::Component
|
11
12
|
|
12
13
|
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
13
14
|
|
@@ -15,18 +16,18 @@ module Sidekiq
|
|
15
16
|
proc { "sidekiq" },
|
16
17
|
proc { Sidekiq::VERSION },
|
17
18
|
proc { |me, data| data["tag"] },
|
18
|
-
proc { |me, data| "[#{Processor::
|
19
|
+
proc { |me, data| "[#{Processor::WORK_STATE.size} of #{data["concurrency"]} busy]" },
|
19
20
|
proc { |me, data| "stopping" if me.stopping? }
|
20
21
|
]
|
21
22
|
|
22
23
|
attr_accessor :manager, :poller, :fetcher
|
23
24
|
|
24
25
|
def initialize(options)
|
26
|
+
@config = options
|
25
27
|
options[:fetch] ||= BasicFetch.new(options)
|
26
28
|
@manager = Sidekiq::Manager.new(options)
|
27
|
-
@poller = Sidekiq::Scheduled::Poller.new
|
29
|
+
@poller = Sidekiq::Scheduled::Poller.new(options)
|
28
30
|
@done = false
|
29
|
-
@options = options
|
30
31
|
end
|
31
32
|
|
32
33
|
def run
|
@@ -43,11 +44,9 @@ module Sidekiq
|
|
43
44
|
@poller.terminate
|
44
45
|
end
|
45
46
|
|
46
|
-
# Shuts down
|
47
|
-
# return until all work is complete and cleaned up.
|
48
|
-
# It can take up to the timeout to complete.
|
47
|
+
# Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
|
49
48
|
def stop
|
50
|
-
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @
|
49
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
|
51
50
|
|
52
51
|
@done = true
|
53
52
|
@manager.quiet
|
@@ -55,10 +54,10 @@ module Sidekiq
|
|
55
54
|
|
56
55
|
@manager.stop(deadline)
|
57
56
|
|
58
|
-
# Requeue everything in case there was a
|
57
|
+
# Requeue everything in case there was a thread which fetched a job while the process was stopped.
|
59
58
|
# This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
|
60
|
-
strategy = @
|
61
|
-
strategy.bulk_requeue([], @
|
59
|
+
strategy = @config[:fetch]
|
60
|
+
strategy.bulk_requeue([], @config)
|
62
61
|
|
63
62
|
clear_heartbeat
|
64
63
|
end
|
@@ -69,22 +68,24 @@ module Sidekiq
|
|
69
68
|
|
70
69
|
private unless $TESTING
|
71
70
|
|
71
|
+
BEAT_PAUSE = 5
|
72
|
+
|
72
73
|
def start_heartbeat
|
73
74
|
loop do
|
74
75
|
heartbeat
|
75
|
-
sleep
|
76
|
+
sleep BEAT_PAUSE
|
76
77
|
end
|
77
|
-
|
78
|
+
logger.info("Heartbeat stopping...")
|
78
79
|
end
|
79
80
|
|
80
81
|
def clear_heartbeat
|
81
82
|
# Remove record from Redis since we are shutting down.
|
82
83
|
# Note we don't stop the heartbeat thread; if the process
|
83
84
|
# doesn't actually exit, it'll reappear in the Web UI.
|
84
|
-
|
85
|
-
conn.pipelined do
|
86
|
-
|
87
|
-
|
85
|
+
redis do |conn|
|
86
|
+
conn.pipelined do |pipeline|
|
87
|
+
pipeline.srem("processes", identity)
|
88
|
+
pipeline.unlink("#{identity}:work")
|
88
89
|
end
|
89
90
|
end
|
90
91
|
rescue
|
@@ -105,14 +106,14 @@ module Sidekiq
|
|
105
106
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
106
107
|
begin
|
107
108
|
Sidekiq.redis do |conn|
|
108
|
-
conn.pipelined do
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
109
|
+
conn.pipelined do |pipeline|
|
110
|
+
pipeline.incrby("stat:processed", procd)
|
111
|
+
pipeline.incrby("stat:processed:#{nowdate}", procd)
|
112
|
+
pipeline.expire("stat:processed:#{nowdate}", STATS_TTL)
|
113
|
+
|
114
|
+
pipeline.incrby("stat:failed", fails)
|
115
|
+
pipeline.incrby("stat:failed:#{nowdate}", fails)
|
116
|
+
pipeline.expire("stat:failed:#{nowdate}", STATS_TTL)
|
116
117
|
end
|
117
118
|
end
|
118
119
|
rescue => ex
|
@@ -130,26 +131,29 @@ module Sidekiq
|
|
130
131
|
begin
|
131
132
|
fails = Processor::FAILURE.reset
|
132
133
|
procd = Processor::PROCESSED.reset
|
133
|
-
curstate = Processor::
|
134
|
+
curstate = Processor::WORK_STATE.dup
|
134
135
|
|
135
|
-
workers_key = "#{key}:workers"
|
136
136
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
137
137
|
|
138
|
-
|
139
|
-
conn.multi do
|
140
|
-
|
141
|
-
|
142
|
-
|
138
|
+
redis do |conn|
|
139
|
+
conn.multi do |transaction|
|
140
|
+
transaction.incrby("stat:processed", procd)
|
141
|
+
transaction.incrby("stat:processed:#{nowdate}", procd)
|
142
|
+
transaction.expire("stat:processed:#{nowdate}", STATS_TTL)
|
143
143
|
|
144
|
-
|
145
|
-
|
146
|
-
|
144
|
+
transaction.incrby("stat:failed", fails)
|
145
|
+
transaction.incrby("stat:failed:#{nowdate}", fails)
|
146
|
+
transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
|
147
|
+
end
|
147
148
|
|
148
|
-
|
149
|
+
# work is the current set of executing jobs
|
150
|
+
work_key = "#{key}:work"
|
151
|
+
conn.pipelined do |transaction|
|
152
|
+
transaction.unlink(work_key)
|
149
153
|
curstate.each_pair do |tid, hash|
|
150
|
-
|
154
|
+
transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
|
151
155
|
end
|
152
|
-
|
156
|
+
transaction.expire(work_key, 60)
|
153
157
|
end
|
154
158
|
end
|
155
159
|
|
@@ -158,18 +162,18 @@ module Sidekiq
|
|
158
162
|
fails = procd = 0
|
159
163
|
kb = memory_usage(::Process.pid)
|
160
164
|
|
161
|
-
_, exists, _, _, msg =
|
162
|
-
conn.multi {
|
163
|
-
|
164
|
-
|
165
|
-
|
165
|
+
_, exists, _, _, msg = redis { |conn|
|
166
|
+
conn.multi { |transaction|
|
167
|
+
transaction.sadd("processes", key)
|
168
|
+
transaction.exists?(key)
|
169
|
+
transaction.hmset(key, "info", to_json,
|
166
170
|
"busy", curstate.size,
|
167
171
|
"beat", Time.now.to_f,
|
168
172
|
"rtt_us", rtt,
|
169
|
-
"quiet", @done,
|
173
|
+
"quiet", @done.to_s,
|
170
174
|
"rss", kb)
|
171
|
-
|
172
|
-
|
175
|
+
transaction.expire(key, 60)
|
176
|
+
transaction.rpop("#{key}-signals")
|
173
177
|
}
|
174
178
|
}
|
175
179
|
|
@@ -196,7 +200,7 @@ module Sidekiq
|
|
196
200
|
|
197
201
|
def check_rtt
|
198
202
|
a = b = 0
|
199
|
-
|
203
|
+
redis do |x|
|
200
204
|
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
201
205
|
x.ping
|
202
206
|
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
@@ -207,10 +211,12 @@ module Sidekiq
|
|
207
211
|
# Workable is < 10,000µs
|
208
212
|
# Log a warning if it's a disaster.
|
209
213
|
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
210
|
-
|
214
|
+
logger.warn <<~EOM
|
211
215
|
Your Redis network connection is performing extremely poorly.
|
212
216
|
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
213
217
|
Ensure Redis is running in the same AZ or datacenter as Sidekiq.
|
218
|
+
If these values are close to 100,000, that means your Sidekiq process may be
|
219
|
+
CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
|
214
220
|
EOM
|
215
221
|
RTT_READINGS.reset
|
216
222
|
end
|
@@ -238,26 +244,22 @@ module Sidekiq
|
|
238
244
|
end
|
239
245
|
|
240
246
|
def to_data
|
241
|
-
@data ||=
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
}
|
252
|
-
end
|
247
|
+
@data ||= {
|
248
|
+
"hostname" => hostname,
|
249
|
+
"started_at" => Time.now.to_f,
|
250
|
+
"pid" => ::Process.pid,
|
251
|
+
"tag" => @config[:tag] || "",
|
252
|
+
"concurrency" => @config[:concurrency],
|
253
|
+
"queues" => @config[:queues].uniq,
|
254
|
+
"labels" => @config[:labels],
|
255
|
+
"identity" => identity
|
256
|
+
}
|
253
257
|
end
|
254
258
|
|
255
259
|
def to_json
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
Sidekiq.dump_json(to_data)
|
260
|
-
end
|
260
|
+
# this data changes infrequently so dump it to a string
|
261
|
+
# now so we don't need to dump it every heartbeat.
|
262
|
+
@json ||= Sidekiq.dump_json(to_data)
|
261
263
|
end
|
262
264
|
end
|
263
265
|
end
|