sidekiq 6.3.1 → 7.0.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +140 -4
- data/LICENSE.txt +9 -0
- data/README.md +19 -13
- data/bin/sidekiq +4 -9
- data/bin/sidekiqload +71 -76
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +267 -186
- data/lib/sidekiq/capsule.rb +110 -0
- data/lib/sidekiq/cli.rb +82 -78
- data/lib/sidekiq/client.rb +73 -80
- data/lib/sidekiq/{util.rb → component.rb} +13 -42
- data/lib/sidekiq/config.rb +271 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +22 -21
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +79 -56
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +76 -82
- data/lib/sidekiq/logger.rb +9 -44
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +84 -42
- data/lib/sidekiq/middleware/current_attributes.rb +19 -13
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +16 -8
- data/lib/sidekiq/processor.rb +56 -59
- data/lib/sidekiq/rails.rb +10 -9
- data/lib/sidekiq/redis_client_adapter.rb +118 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +75 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +27 -8
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +22 -20
- data/lib/sidekiq/web.rb +6 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +85 -202
- data/sidekiq.gemspec +29 -5
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +236 -0
- data/web/assets/stylesheets/application-dark.css +13 -17
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +67 -300
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +52 -52
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +71 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +37 -11
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +9 -4
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +80 -0
- data/web/views/metrics_for_job.erb +69 -0
- data/web/views/queue.erb +5 -1
- metadata +75 -27
- data/LICENSE +0 -9
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/delay.rb +0 -41
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/worker.rb +0 -311
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/scheduled"
|
4
|
-
require "sidekiq/api"
|
5
|
-
|
6
3
|
require "zlib"
|
7
4
|
require "base64"
|
5
|
+
require "sidekiq/component"
|
8
6
|
|
9
7
|
module Sidekiq
|
10
8
|
##
|
@@ -25,18 +23,19 @@ module Sidekiq
|
|
25
23
|
#
|
26
24
|
# A job looks like:
|
27
25
|
#
|
28
|
-
# { 'class' => '
|
26
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
|
29
27
|
#
|
30
28
|
# The 'retry' option also accepts a number (in place of 'true'):
|
31
29
|
#
|
32
|
-
# { 'class' => '
|
30
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
|
33
31
|
#
|
34
32
|
# The job will be retried this number of times before giving up. (If simply
|
35
33
|
# 'true', Sidekiq retries 25 times)
|
36
34
|
#
|
37
|
-
#
|
35
|
+
# Relevant options for job retries:
|
38
36
|
#
|
39
|
-
# * 'queue' - the queue
|
37
|
+
# * 'queue' - the queue for the initial job
|
38
|
+
# * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
|
40
39
|
# * 'retry_count' - number of times we've retried so far.
|
41
40
|
# * 'error_message' - the message from the exception
|
42
41
|
# * 'error_class' - the exception class
|
@@ -52,11 +51,12 @@ module Sidekiq
|
|
52
51
|
#
|
53
52
|
# Sidekiq.options[:max_retries] = 7
|
54
53
|
#
|
55
|
-
# or limit the number of retries for a particular
|
54
|
+
# or limit the number of retries for a particular job and send retries to
|
55
|
+
# a low priority queue with:
|
56
56
|
#
|
57
|
-
# class
|
58
|
-
# include Sidekiq::
|
59
|
-
# sidekiq_options :
|
57
|
+
# class MyJob
|
58
|
+
# include Sidekiq::Job
|
59
|
+
# sidekiq_options retry: 10, retry_queue: 'low'
|
60
60
|
# end
|
61
61
|
#
|
62
62
|
class JobRetry
|
@@ -64,17 +64,18 @@ module Sidekiq
|
|
64
64
|
|
65
65
|
class Skip < Handled; end
|
66
66
|
|
67
|
-
include Sidekiq::
|
67
|
+
include Sidekiq::Component
|
68
68
|
|
69
69
|
DEFAULT_MAX_RETRY_ATTEMPTS = 25
|
70
70
|
|
71
|
-
def initialize(
|
72
|
-
@
|
71
|
+
def initialize(capsule)
|
72
|
+
@config = @capsule = capsule
|
73
|
+
@max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
|
73
74
|
end
|
74
75
|
|
75
76
|
# The global retry handler requires only the barest of data.
|
76
77
|
# We want to be able to retry as much as possible so we don't
|
77
|
-
# require the
|
78
|
+
# require the job to be instantiated.
|
78
79
|
def global(jobstr, queue)
|
79
80
|
yield
|
80
81
|
rescue Handled => ex
|
@@ -88,9 +89,9 @@ module Sidekiq
|
|
88
89
|
|
89
90
|
msg = Sidekiq.load_json(jobstr)
|
90
91
|
if msg["retry"]
|
91
|
-
|
92
|
+
process_retry(nil, msg, queue, e)
|
92
93
|
else
|
93
|
-
|
94
|
+
@capsule.config.death_handlers.each do |handler|
|
94
95
|
handler.call(msg, e)
|
95
96
|
rescue => handler_ex
|
96
97
|
handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
|
@@ -101,14 +102,14 @@ module Sidekiq
|
|
101
102
|
end
|
102
103
|
|
103
104
|
# The local retry support means that any errors that occur within
|
104
|
-
# this block can be associated with the given
|
105
|
+
# this block can be associated with the given job instance.
|
105
106
|
# This is required to support the `sidekiq_retries_exhausted` block.
|
106
107
|
#
|
107
108
|
# Note that any exception from the block is wrapped in the Skip
|
108
109
|
# exception so the global block does not reprocess the error. The
|
109
110
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
110
111
|
# calling the handle_exception handlers.
|
111
|
-
def local(
|
112
|
+
def local(jobinst, jobstr, queue)
|
112
113
|
yield
|
113
114
|
rescue Handled => ex
|
114
115
|
raise ex
|
@@ -121,11 +122,11 @@ module Sidekiq
|
|
121
122
|
|
122
123
|
msg = Sidekiq.load_json(jobstr)
|
123
124
|
if msg["retry"].nil?
|
124
|
-
msg["retry"] =
|
125
|
+
msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
|
125
126
|
end
|
126
127
|
|
127
128
|
raise e unless msg["retry"]
|
128
|
-
|
129
|
+
process_retry(jobinst, msg, queue, e)
|
129
130
|
# We've handled this error associated with this job, don't
|
130
131
|
# need to handle it at the global level
|
131
132
|
raise Skip
|
@@ -133,10 +134,10 @@ module Sidekiq
|
|
133
134
|
|
134
135
|
private
|
135
136
|
|
136
|
-
# Note that +
|
137
|
-
# instantiate the
|
137
|
+
# Note that +jobinst+ can be nil here if an error is raised before we can
|
138
|
+
# instantiate the job instance. All access must be guarded and
|
138
139
|
# best effort.
|
139
|
-
def
|
140
|
+
def process_retry(jobinst, msg, queue, exception)
|
140
141
|
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
141
142
|
|
142
143
|
msg["queue"] = (msg["retry_queue"] || queue)
|
@@ -167,24 +168,54 @@ module Sidekiq
|
|
167
168
|
msg["error_backtrace"] = compress_backtrace(lines)
|
168
169
|
end
|
169
170
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
171
|
+
# Goodbye dear message, you (re)tried your best I'm sure.
|
172
|
+
return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
|
173
|
+
|
174
|
+
strategy, delay = delay_for(jobinst, count, exception)
|
175
|
+
case strategy
|
176
|
+
when :discard
|
177
|
+
return # poof!
|
178
|
+
when :kill
|
179
|
+
return retries_exhausted(jobinst, msg, exception)
|
180
|
+
end
|
181
|
+
|
182
|
+
# Logging here can break retries if the logging device raises ENOSPC #3979
|
183
|
+
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
184
|
+
jitter = rand(10) * (count + 1)
|
185
|
+
retry_at = Time.now.to_f + delay + jitter
|
186
|
+
payload = Sidekiq.dump_json(msg)
|
187
|
+
redis do |conn|
|
188
|
+
conn.zadd("retry", retry_at.to_s, payload)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
# returns (strategy, seconds)
|
193
|
+
def delay_for(jobinst, count, exception)
|
194
|
+
rv = begin
|
195
|
+
# sidekiq_retry_in can return two different things:
|
196
|
+
# 1. When to retry next, as an integer of seconds
|
197
|
+
# 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
|
198
|
+
jobinst&.sidekiq_retry_in_block&.call(count, exception)
|
199
|
+
rescue Exception => e
|
200
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
201
|
+
nil
|
202
|
+
end
|
203
|
+
|
204
|
+
delay = (count**4) + 15
|
205
|
+
if Integer === rv && rv > 0
|
206
|
+
delay = rv
|
207
|
+
elsif rv == :discard
|
208
|
+
return [:discard, nil] # do nothing, job goes poof
|
209
|
+
elsif rv == :kill
|
210
|
+
return [:kill, nil]
|
182
211
|
end
|
212
|
+
|
213
|
+
[:default, delay]
|
183
214
|
end
|
184
215
|
|
185
|
-
def retries_exhausted(
|
216
|
+
def retries_exhausted(jobinst, msg, exception)
|
186
217
|
begin
|
187
|
-
block =
|
218
|
+
block = jobinst&.sidekiq_retries_exhausted_block
|
188
219
|
block&.call(msg, exception)
|
189
220
|
rescue => e
|
190
221
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
@@ -192,7 +223,7 @@ module Sidekiq
|
|
192
223
|
|
193
224
|
send_to_morgue(msg) unless msg["dead"] == false
|
194
225
|
|
195
|
-
|
226
|
+
@capsule.config.death_handlers.each do |handler|
|
196
227
|
handler.call(msg, exception)
|
197
228
|
rescue => e
|
198
229
|
handle_exception(e, {context: "Error calling death handler", job: msg})
|
@@ -202,7 +233,15 @@ module Sidekiq
|
|
202
233
|
def send_to_morgue(msg)
|
203
234
|
logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
|
204
235
|
payload = Sidekiq.dump_json(msg)
|
205
|
-
|
236
|
+
now = Time.now.to_f
|
237
|
+
|
238
|
+
redis do |conn|
|
239
|
+
conn.multi do |xa|
|
240
|
+
xa.zadd("dead", now.to_s, payload)
|
241
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
242
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
243
|
+
end
|
244
|
+
end
|
206
245
|
end
|
207
246
|
|
208
247
|
def retry_attempts_from(msg_retry, default)
|
@@ -213,22 +252,6 @@ module Sidekiq
|
|
213
252
|
end
|
214
253
|
end
|
215
254
|
|
216
|
-
def delay_for(worker, count, exception)
|
217
|
-
jitter = rand(10) * (count + 1)
|
218
|
-
if worker&.sidekiq_retry_in_block
|
219
|
-
custom_retry_in = retry_in(worker, count, exception).to_i
|
220
|
-
return custom_retry_in + jitter if custom_retry_in > 0
|
221
|
-
end
|
222
|
-
(count**4) + 15 + jitter
|
223
|
-
end
|
224
|
-
|
225
|
-
def retry_in(worker, count, exception)
|
226
|
-
worker.sidekiq_retry_in_block.call(count, exception)
|
227
|
-
rescue Exception => e
|
228
|
-
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
|
229
|
-
nil
|
230
|
-
end
|
231
|
-
|
232
255
|
def exception_caused_by_shutdown?(e, checked_causes = [])
|
233
256
|
return false unless e.cause
|
234
257
|
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require "securerandom"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module JobUtil
|
6
|
+
# These functions encapsulate various job utilities.
|
7
|
+
|
8
|
+
TRANSIENT_ATTRIBUTES = %w[]
|
9
|
+
|
10
|
+
def validate(item)
|
11
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
12
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
13
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
14
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
15
|
+
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
16
|
+
end
|
17
|
+
|
18
|
+
def verify_json(item)
|
19
|
+
job_class = item["wrapped"] || item["class"]
|
20
|
+
if Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :raise
|
21
|
+
msg = <<~EOM
|
22
|
+
Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
|
23
|
+
To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
|
24
|
+
EOM
|
25
|
+
raise(ArgumentError, msg) unless json_safe?(item)
|
26
|
+
elsif Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :warn
|
27
|
+
warn <<~EOM unless json_safe?(item)
|
28
|
+
Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
|
29
|
+
Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
|
30
|
+
by calling `Sidekiq.strict_args!` during Sidekiq initialization.
|
31
|
+
EOM
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def normalize_item(item)
|
36
|
+
validate(item)
|
37
|
+
|
38
|
+
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
39
|
+
# this allows ActiveJobs to control sidekiq_options too.
|
40
|
+
defaults = normalized_hash(item["class"])
|
41
|
+
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
|
42
|
+
item = defaults.merge(item)
|
43
|
+
|
44
|
+
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
45
|
+
|
46
|
+
# remove job attributes which aren't necessary to persist into Redis
|
47
|
+
TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
|
48
|
+
|
49
|
+
item["jid"] ||= SecureRandom.hex(12)
|
50
|
+
item["class"] = item["class"].to_s
|
51
|
+
item["queue"] = item["queue"].to_s
|
52
|
+
item["created_at"] ||= Time.now.to_f
|
53
|
+
item
|
54
|
+
end
|
55
|
+
|
56
|
+
def normalized_hash(item_class)
|
57
|
+
if item_class.is_a?(Class)
|
58
|
+
raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
|
59
|
+
item_class.get_sidekiq_options
|
60
|
+
else
|
61
|
+
Sidekiq.default_job_options
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def json_safe?(item)
|
68
|
+
JSON.parse(JSON.dump(item["args"])) == item["args"]
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq/manager"
|
4
|
-
require "sidekiq/
|
4
|
+
require "sidekiq/capsule"
|
5
5
|
require "sidekiq/scheduled"
|
6
|
+
require "sidekiq/ring_buffer"
|
6
7
|
|
7
8
|
module Sidekiq
|
8
|
-
# The Launcher starts the
|
9
|
+
# The Launcher starts the Capsule Managers, the Poller thread and provides the process heartbeat.
|
9
10
|
class Launcher
|
10
|
-
include
|
11
|
+
include Sidekiq::Component
|
11
12
|
|
12
13
|
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
13
14
|
|
@@ -15,50 +16,53 @@ module Sidekiq
|
|
15
16
|
proc { "sidekiq" },
|
16
17
|
proc { Sidekiq::VERSION },
|
17
18
|
proc { |me, data| data["tag"] },
|
18
|
-
proc { |me, data| "[#{Processor::
|
19
|
+
proc { |me, data| "[#{Processor::WORK_STATE.size} of #{me.config.total_concurrency} busy]" },
|
19
20
|
proc { |me, data| "stopping" if me.stopping? }
|
20
21
|
]
|
21
22
|
|
22
|
-
attr_accessor :
|
23
|
+
attr_accessor :managers, :poller
|
23
24
|
|
24
|
-
def initialize(
|
25
|
-
|
26
|
-
@
|
27
|
-
@
|
25
|
+
def initialize(config, embedded: false)
|
26
|
+
@config = config
|
27
|
+
@embedded = embedded
|
28
|
+
@managers = config.capsules.values.map do |cap|
|
29
|
+
Sidekiq::Manager.new(cap)
|
30
|
+
end
|
31
|
+
@poller = Sidekiq::Scheduled::Poller.new(@config)
|
28
32
|
@done = false
|
29
|
-
@options = options
|
30
33
|
end
|
31
34
|
|
32
35
|
def run
|
36
|
+
Sidekiq.freeze!
|
33
37
|
@thread = safe_thread("heartbeat", &method(:start_heartbeat))
|
34
38
|
@poller.start
|
35
|
-
@
|
39
|
+
@managers.each(&:start)
|
36
40
|
end
|
37
41
|
|
38
42
|
# Stops this instance from processing any more jobs,
|
39
43
|
#
|
40
44
|
def quiet
|
45
|
+
return if @done
|
46
|
+
|
41
47
|
@done = true
|
42
|
-
@
|
48
|
+
@managers.each(&:quiet)
|
43
49
|
@poller.terminate
|
50
|
+
fire_event(:quiet, reverse: true)
|
44
51
|
end
|
45
52
|
|
46
|
-
# Shuts down
|
47
|
-
# return until all work is complete and cleaned up.
|
48
|
-
# It can take up to the timeout to complete.
|
53
|
+
# Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
|
49
54
|
def stop
|
50
|
-
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @
|
51
|
-
|
52
|
-
@done = true
|
53
|
-
@manager.quiet
|
54
|
-
@poller.terminate
|
55
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
|
55
56
|
|
56
|
-
|
57
|
+
quiet
|
58
|
+
stoppers = @managers.map do |mgr|
|
59
|
+
Thread.new do
|
60
|
+
mgr.stop(deadline)
|
61
|
+
end
|
62
|
+
end
|
57
63
|
|
58
|
-
|
59
|
-
|
60
|
-
strategy = @options[:fetch]
|
61
|
-
strategy.bulk_requeue([], @options)
|
64
|
+
fire_event(:shutdown, reverse: true)
|
65
|
+
stoppers.each(&:join)
|
62
66
|
|
63
67
|
clear_heartbeat
|
64
68
|
end
|
@@ -69,24 +73,26 @@ module Sidekiq
|
|
69
73
|
|
70
74
|
private unless $TESTING
|
71
75
|
|
72
|
-
BEAT_PAUSE =
|
76
|
+
BEAT_PAUSE = 10
|
73
77
|
|
74
78
|
def start_heartbeat
|
75
79
|
loop do
|
76
80
|
heartbeat
|
77
81
|
sleep BEAT_PAUSE
|
78
82
|
end
|
79
|
-
|
83
|
+
logger.info("Heartbeat stopping...")
|
80
84
|
end
|
81
85
|
|
82
86
|
def clear_heartbeat
|
87
|
+
flush_stats
|
88
|
+
|
83
89
|
# Remove record from Redis since we are shutting down.
|
84
90
|
# Note we don't stop the heartbeat thread; if the process
|
85
91
|
# doesn't actually exit, it'll reappear in the Web UI.
|
86
|
-
|
87
|
-
conn.pipelined do
|
88
|
-
|
89
|
-
|
92
|
+
redis do |conn|
|
93
|
+
conn.pipelined do |pipeline|
|
94
|
+
pipeline.srem("processes", [identity])
|
95
|
+
pipeline.unlink("#{identity}:work")
|
90
96
|
end
|
91
97
|
end
|
92
98
|
rescue
|
@@ -94,64 +100,51 @@ module Sidekiq
|
|
94
100
|
end
|
95
101
|
|
96
102
|
def heartbeat
|
97
|
-
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
|
103
|
+
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ") unless @embedded
|
98
104
|
|
99
105
|
❤
|
100
106
|
end
|
101
107
|
|
102
|
-
def
|
108
|
+
def flush_stats
|
103
109
|
fails = Processor::FAILURE.reset
|
104
110
|
procd = Processor::PROCESSED.reset
|
105
111
|
return if fails + procd == 0
|
106
112
|
|
107
113
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
108
114
|
begin
|
109
|
-
|
110
|
-
conn.pipelined do
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
115
|
+
redis do |conn|
|
116
|
+
conn.pipelined do |pipeline|
|
117
|
+
pipeline.incrby("stat:processed", procd)
|
118
|
+
pipeline.incrby("stat:processed:#{nowdate}", procd)
|
119
|
+
pipeline.expire("stat:processed:#{nowdate}", STATS_TTL)
|
120
|
+
|
121
|
+
pipeline.incrby("stat:failed", fails)
|
122
|
+
pipeline.incrby("stat:failed:#{nowdate}", fails)
|
123
|
+
pipeline.expire("stat:failed:#{nowdate}", STATS_TTL)
|
118
124
|
end
|
119
125
|
end
|
120
126
|
rescue => ex
|
121
|
-
|
122
|
-
# try to handle the exception
|
123
|
-
Sidekiq.logger.warn("Unable to flush stats: #{ex}")
|
127
|
+
logger.warn("Unable to flush stats: #{ex}")
|
124
128
|
end
|
125
129
|
end
|
126
|
-
at_exit(&method(:flush_stats))
|
127
130
|
|
128
131
|
def ❤
|
129
132
|
key = identity
|
130
133
|
fails = procd = 0
|
131
134
|
|
132
135
|
begin
|
133
|
-
|
134
|
-
|
135
|
-
curstate = Processor::
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
conn.multi do
|
142
|
-
conn.incrby("stat:processed", procd)
|
143
|
-
conn.incrby("stat:processed:#{nowdate}", procd)
|
144
|
-
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
145
|
-
|
146
|
-
conn.incrby("stat:failed", fails)
|
147
|
-
conn.incrby("stat:failed:#{nowdate}", fails)
|
148
|
-
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
149
|
-
|
150
|
-
conn.unlink(workers_key)
|
136
|
+
flush_stats
|
137
|
+
|
138
|
+
curstate = Processor::WORK_STATE.dup
|
139
|
+
redis do |conn|
|
140
|
+
# work is the current set of executing jobs
|
141
|
+
work_key = "#{key}:work"
|
142
|
+
conn.pipelined do |transaction|
|
143
|
+
transaction.unlink(work_key)
|
151
144
|
curstate.each_pair do |tid, hash|
|
152
|
-
|
145
|
+
transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
|
153
146
|
end
|
154
|
-
|
147
|
+
transaction.expire(work_key, 60)
|
155
148
|
end
|
156
149
|
end
|
157
150
|
|
@@ -160,23 +153,24 @@ module Sidekiq
|
|
160
153
|
fails = procd = 0
|
161
154
|
kb = memory_usage(::Process.pid)
|
162
155
|
|
163
|
-
_, exists, _, _, msg =
|
164
|
-
conn.multi {
|
165
|
-
|
166
|
-
|
167
|
-
|
156
|
+
_, exists, _, _, msg = redis { |conn|
|
157
|
+
conn.multi { |transaction|
|
158
|
+
transaction.sadd("processes", [key])
|
159
|
+
transaction.exists(key)
|
160
|
+
transaction.hmset(key, "info", to_json,
|
168
161
|
"busy", curstate.size,
|
169
162
|
"beat", Time.now.to_f,
|
170
163
|
"rtt_us", rtt,
|
171
|
-
"quiet", @done,
|
164
|
+
"quiet", @done.to_s,
|
172
165
|
"rss", kb)
|
173
|
-
|
174
|
-
|
166
|
+
transaction.expire(key, 60)
|
167
|
+
transaction.rpop("#{key}-signals")
|
175
168
|
}
|
176
169
|
}
|
177
170
|
|
178
171
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
179
|
-
fire_event(:heartbeat) unless exists
|
172
|
+
fire_event(:heartbeat) unless exists > 0
|
173
|
+
fire_event(:beat, oneshot: false)
|
180
174
|
|
181
175
|
return unless msg
|
182
176
|
|
@@ -198,7 +192,7 @@ module Sidekiq
|
|
198
192
|
|
199
193
|
def check_rtt
|
200
194
|
a = b = 0
|
201
|
-
|
195
|
+
redis do |x|
|
202
196
|
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
203
197
|
x.ping
|
204
198
|
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
@@ -209,12 +203,12 @@ module Sidekiq
|
|
209
203
|
# Workable is < 10,000µs
|
210
204
|
# Log a warning if it's a disaster.
|
211
205
|
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
212
|
-
|
206
|
+
logger.warn <<~EOM
|
213
207
|
Your Redis network connection is performing extremely poorly.
|
214
208
|
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
215
209
|
Ensure Redis is running in the same AZ or datacenter as Sidekiq.
|
216
210
|
If these values are close to 100,000, that means your Sidekiq process may be
|
217
|
-
CPU
|
211
|
+
CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
|
218
212
|
EOM
|
219
213
|
RTT_READINGS.reset
|
220
214
|
end
|
@@ -246,10 +240,10 @@ module Sidekiq
|
|
246
240
|
"hostname" => hostname,
|
247
241
|
"started_at" => Time.now.to_f,
|
248
242
|
"pid" => ::Process.pid,
|
249
|
-
"tag" => @
|
250
|
-
"concurrency" => @
|
251
|
-
"queues" => @
|
252
|
-
"labels" => @
|
243
|
+
"tag" => @config[:tag] || "",
|
244
|
+
"concurrency" => @config.total_concurrency,
|
245
|
+
"queues" => @config.capsules.values.map { |cap| cap.queues }.flatten.uniq,
|
246
|
+
"labels" => @config[:labels].to_a,
|
253
247
|
"identity" => identity
|
254
248
|
}
|
255
249
|
end
|
data/lib/sidekiq/logger.rb
CHANGED
@@ -16,6 +16,10 @@ module Sidekiq
|
|
16
16
|
def self.current
|
17
17
|
Thread.current[:sidekiq_context] ||= {}
|
18
18
|
end
|
19
|
+
|
20
|
+
def self.add(k, v)
|
21
|
+
current[k] = v
|
22
|
+
end
|
19
23
|
end
|
20
24
|
|
21
25
|
module LoggingUtils
|
@@ -27,28 +31,14 @@ module Sidekiq
|
|
27
31
|
"fatal" => 4
|
28
32
|
}
|
29
33
|
LEVELS.default_proc = proc do |_, level|
|
30
|
-
|
34
|
+
puts("Invalid log level: #{level.inspect}")
|
31
35
|
nil
|
32
36
|
end
|
33
37
|
|
34
|
-
|
35
|
-
level
|
36
|
-
|
37
|
-
|
38
|
-
def info?
|
39
|
-
level <= 1
|
40
|
-
end
|
41
|
-
|
42
|
-
def warn?
|
43
|
-
level <= 2
|
44
|
-
end
|
45
|
-
|
46
|
-
def error?
|
47
|
-
level <= 3
|
48
|
-
end
|
49
|
-
|
50
|
-
def fatal?
|
51
|
-
level <= 4
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
52
42
|
end
|
53
43
|
|
54
44
|
def local_level
|
@@ -80,36 +70,11 @@ module Sidekiq
|
|
80
70
|
ensure
|
81
71
|
self.local_level = old_local_level
|
82
72
|
end
|
83
|
-
|
84
|
-
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
85
|
-
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
86
|
-
def add(severity, message = nil, progname = nil, &block)
|
87
|
-
severity ||= ::Logger::UNKNOWN
|
88
|
-
progname ||= @progname
|
89
|
-
|
90
|
-
return true if @logdev.nil? || severity < level
|
91
|
-
|
92
|
-
if message.nil?
|
93
|
-
if block
|
94
|
-
message = yield
|
95
|
-
else
|
96
|
-
message = progname
|
97
|
-
progname = @progname
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
102
|
-
end
|
103
73
|
end
|
104
74
|
|
105
75
|
class Logger < ::Logger
|
106
76
|
include LoggingUtils
|
107
77
|
|
108
|
-
def initialize(*args, **kwargs)
|
109
|
-
super
|
110
|
-
self.formatter = Sidekiq.log_formatter
|
111
|
-
end
|
112
|
-
|
113
78
|
module Formatters
|
114
79
|
class Base < ::Logger::Formatter
|
115
80
|
def tid
|