sidekiq 6.4.1 → 7.2.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +307 -12
- data/README.md +43 -35
- data/bin/multi_queue_bench +268 -0
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +206 -114
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +356 -167
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +85 -89
- data/lib/sidekiq/client.rb +87 -59
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +21 -22
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +97 -58
- data/lib/sidekiq/job_util.rb +62 -20
- data/lib/sidekiq/launcher.rb +91 -83
- data/lib/sidekiq/logger.rb +6 -45
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +156 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +140 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +58 -20
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +11 -3
- data/lib/sidekiq/processor.rb +81 -80
- data/lib/sidekiq/rails.rb +21 -14
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +16 -85
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +66 -38
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +67 -75
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +107 -10
- data/lib/sidekiq/web/csrf_protection.rb +8 -5
- data/lib/sidekiq/web/helpers.rb +65 -43
- data/lib/sidekiq/web.rb +19 -14
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -207
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +92 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +10 -249
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +75 -299
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +84 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +50 -34
- data/web/views/dashboard.erb +26 -4
- data/web/views/filtering.erb +7 -0
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +9 -3
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +58 -27
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -362
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/launcher.rb
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq/manager"
|
4
|
-
require "sidekiq/
|
4
|
+
require "sidekiq/capsule"
|
5
5
|
require "sidekiq/scheduled"
|
6
|
+
require "sidekiq/ring_buffer"
|
6
7
|
|
7
8
|
module Sidekiq
|
8
|
-
# The Launcher starts the
|
9
|
+
# The Launcher starts the Capsule Managers, the Poller thread and provides the process heartbeat.
|
9
10
|
class Launcher
|
10
|
-
include
|
11
|
+
include Sidekiq::Component
|
11
12
|
|
12
13
|
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
13
14
|
|
@@ -15,50 +16,56 @@ module Sidekiq
|
|
15
16
|
proc { "sidekiq" },
|
16
17
|
proc { Sidekiq::VERSION },
|
17
18
|
proc { |me, data| data["tag"] },
|
18
|
-
proc { |me, data| "[#{Processor::
|
19
|
+
proc { |me, data| "[#{Processor::WORK_STATE.size} of #{me.config.total_concurrency} busy]" },
|
19
20
|
proc { |me, data| "stopping" if me.stopping? }
|
20
21
|
]
|
21
22
|
|
22
|
-
attr_accessor :
|
23
|
+
attr_accessor :managers, :poller
|
23
24
|
|
24
|
-
def initialize(
|
25
|
-
|
26
|
-
@
|
27
|
-
@
|
25
|
+
def initialize(config, embedded: false)
|
26
|
+
@config = config
|
27
|
+
@embedded = embedded
|
28
|
+
@managers = config.capsules.values.map do |cap|
|
29
|
+
Sidekiq::Manager.new(cap)
|
30
|
+
end
|
31
|
+
@poller = Sidekiq::Scheduled::Poller.new(@config)
|
28
32
|
@done = false
|
29
|
-
@options = options
|
30
33
|
end
|
31
34
|
|
32
|
-
|
33
|
-
|
35
|
+
# Start this Sidekiq instance. If an embedding process already
|
36
|
+
# has a heartbeat thread, caller can use `async_beat: false`
|
37
|
+
# and instead have thread call Launcher#heartbeat every N seconds.
|
38
|
+
def run(async_beat: true)
|
39
|
+
Sidekiq.freeze!
|
40
|
+
logger.debug { @config.merge!({}) }
|
41
|
+
@thread = safe_thread("heartbeat", &method(:start_heartbeat)) if async_beat
|
34
42
|
@poller.start
|
35
|
-
@
|
43
|
+
@managers.each(&:start)
|
36
44
|
end
|
37
45
|
|
38
46
|
# Stops this instance from processing any more jobs,
|
39
|
-
#
|
40
47
|
def quiet
|
48
|
+
return if @done
|
49
|
+
|
41
50
|
@done = true
|
42
|
-
@
|
51
|
+
@managers.each(&:quiet)
|
43
52
|
@poller.terminate
|
53
|
+
fire_event(:quiet, reverse: true)
|
44
54
|
end
|
45
55
|
|
46
|
-
# Shuts down
|
47
|
-
# return until all work is complete and cleaned up.
|
48
|
-
# It can take up to the timeout to complete.
|
56
|
+
# Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
|
49
57
|
def stop
|
50
|
-
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @
|
58
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
|
51
59
|
|
52
|
-
|
53
|
-
@
|
54
|
-
|
55
|
-
|
56
|
-
|
60
|
+
quiet
|
61
|
+
stoppers = @managers.map do |mgr|
|
62
|
+
Thread.new do
|
63
|
+
mgr.stop(deadline)
|
64
|
+
end
|
65
|
+
end
|
57
66
|
|
58
|
-
|
59
|
-
|
60
|
-
strategy = @options[:fetch]
|
61
|
-
strategy.bulk_requeue([], @options)
|
67
|
+
fire_event(:shutdown, reverse: true)
|
68
|
+
stoppers.each(&:join)
|
62
69
|
|
63
70
|
clear_heartbeat
|
64
71
|
end
|
@@ -67,46 +74,54 @@ module Sidekiq
|
|
67
74
|
@done
|
68
75
|
end
|
69
76
|
|
77
|
+
# If embedding Sidekiq, you can have the process heartbeat
|
78
|
+
# call this method to regularly heartbeat rather than creating
|
79
|
+
# a separate thread.
|
80
|
+
def heartbeat
|
81
|
+
❤
|
82
|
+
end
|
83
|
+
|
70
84
|
private unless $TESTING
|
71
85
|
|
72
|
-
BEAT_PAUSE =
|
86
|
+
BEAT_PAUSE = 10
|
73
87
|
|
74
88
|
def start_heartbeat
|
75
89
|
loop do
|
76
|
-
|
90
|
+
beat
|
77
91
|
sleep BEAT_PAUSE
|
78
92
|
end
|
79
|
-
|
93
|
+
logger.info("Heartbeat stopping...")
|
94
|
+
end
|
95
|
+
|
96
|
+
def beat
|
97
|
+
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ") unless @embedded
|
98
|
+
❤
|
80
99
|
end
|
81
100
|
|
82
101
|
def clear_heartbeat
|
102
|
+
flush_stats
|
103
|
+
|
83
104
|
# Remove record from Redis since we are shutting down.
|
84
105
|
# Note we don't stop the heartbeat thread; if the process
|
85
106
|
# doesn't actually exit, it'll reappear in the Web UI.
|
86
|
-
|
107
|
+
redis do |conn|
|
87
108
|
conn.pipelined do |pipeline|
|
88
|
-
pipeline.srem("processes", identity)
|
89
|
-
pipeline.unlink("#{identity}:
|
109
|
+
pipeline.srem("processes", [identity])
|
110
|
+
pipeline.unlink("#{identity}:work")
|
90
111
|
end
|
91
112
|
end
|
92
113
|
rescue
|
93
114
|
# best effort, ignore network errors
|
94
115
|
end
|
95
116
|
|
96
|
-
def
|
97
|
-
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
|
98
|
-
|
99
|
-
❤
|
100
|
-
end
|
101
|
-
|
102
|
-
def self.flush_stats
|
117
|
+
def flush_stats
|
103
118
|
fails = Processor::FAILURE.reset
|
104
119
|
procd = Processor::PROCESSED.reset
|
105
120
|
return if fails + procd == 0
|
106
121
|
|
107
122
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
108
123
|
begin
|
109
|
-
|
124
|
+
redis do |conn|
|
110
125
|
conn.pipelined do |pipeline|
|
111
126
|
pipeline.incrby("stat:processed", procd)
|
112
127
|
pipeline.incrby("stat:processed:#{nowdate}", procd)
|
@@ -118,40 +133,27 @@ module Sidekiq
|
|
118
133
|
end
|
119
134
|
end
|
120
135
|
rescue => ex
|
121
|
-
|
122
|
-
# try to handle the exception
|
123
|
-
Sidekiq.logger.warn("Unable to flush stats: #{ex}")
|
136
|
+
logger.warn("Unable to flush stats: #{ex}")
|
124
137
|
end
|
125
138
|
end
|
126
|
-
at_exit(&method(:flush_stats))
|
127
139
|
|
128
140
|
def ❤
|
129
141
|
key = identity
|
130
142
|
fails = procd = 0
|
131
143
|
|
132
144
|
begin
|
133
|
-
|
134
|
-
|
135
|
-
curstate = Processor::
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
conn.multi do |transaction|
|
142
|
-
transaction.incrby("stat:processed", procd)
|
143
|
-
transaction.incrby("stat:processed:#{nowdate}", procd)
|
144
|
-
transaction.expire("stat:processed:#{nowdate}", STATS_TTL)
|
145
|
-
|
146
|
-
transaction.incrby("stat:failed", fails)
|
147
|
-
transaction.incrby("stat:failed:#{nowdate}", fails)
|
148
|
-
transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
|
149
|
-
|
150
|
-
transaction.unlink(workers_key)
|
145
|
+
flush_stats
|
146
|
+
|
147
|
+
curstate = Processor::WORK_STATE.dup
|
148
|
+
redis do |conn|
|
149
|
+
# work is the current set of executing jobs
|
150
|
+
work_key = "#{key}:work"
|
151
|
+
conn.pipelined do |transaction|
|
152
|
+
transaction.unlink(work_key)
|
151
153
|
curstate.each_pair do |tid, hash|
|
152
|
-
transaction.hset(
|
154
|
+
transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
|
153
155
|
end
|
154
|
-
transaction.expire(
|
156
|
+
transaction.expire(work_key, 60)
|
155
157
|
end
|
156
158
|
end
|
157
159
|
|
@@ -160,15 +162,15 @@ module Sidekiq
|
|
160
162
|
fails = procd = 0
|
161
163
|
kb = memory_usage(::Process.pid)
|
162
164
|
|
163
|
-
_, exists, _, _,
|
165
|
+
_, exists, _, _, signal = redis { |conn|
|
164
166
|
conn.multi { |transaction|
|
165
|
-
transaction.sadd("processes", key)
|
166
|
-
transaction.exists
|
167
|
-
transaction.
|
167
|
+
transaction.sadd("processes", [key])
|
168
|
+
transaction.exists(key)
|
169
|
+
transaction.hset(key, "info", to_json,
|
168
170
|
"busy", curstate.size,
|
169
171
|
"beat", Time.now.to_f,
|
170
172
|
"rtt_us", rtt,
|
171
|
-
"quiet", @done,
|
173
|
+
"quiet", @done.to_s,
|
172
174
|
"rss", kb)
|
173
175
|
transaction.expire(key, 60)
|
174
176
|
transaction.rpop("#{key}-signals")
|
@@ -176,11 +178,10 @@ module Sidekiq
|
|
176
178
|
}
|
177
179
|
|
178
180
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
179
|
-
fire_event(:heartbeat) unless exists
|
181
|
+
fire_event(:heartbeat) unless exists > 0
|
182
|
+
fire_event(:beat, oneshot: false)
|
180
183
|
|
181
|
-
|
182
|
-
|
183
|
-
::Process.kill(msg, ::Process.pid)
|
184
|
+
::Process.kill(signal, ::Process.pid) if signal && !@embedded
|
184
185
|
rescue => e
|
185
186
|
# ignore all redis/network issues
|
186
187
|
logger.error("heartbeat: #{e}")
|
@@ -198,7 +199,7 @@ module Sidekiq
|
|
198
199
|
|
199
200
|
def check_rtt
|
200
201
|
a = b = 0
|
201
|
-
|
202
|
+
redis do |x|
|
202
203
|
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
203
204
|
x.ping
|
204
205
|
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
@@ -209,12 +210,12 @@ module Sidekiq
|
|
209
210
|
# Workable is < 10,000µs
|
210
211
|
# Log a warning if it's a disaster.
|
211
212
|
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
212
|
-
|
213
|
+
logger.warn <<~EOM
|
213
214
|
Your Redis network connection is performing extremely poorly.
|
214
215
|
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
215
216
|
Ensure Redis is running in the same AZ or datacenter as Sidekiq.
|
216
217
|
If these values are close to 100,000, that means your Sidekiq process may be
|
217
|
-
CPU
|
218
|
+
CPU-saturated; reduce your concurrency and/or see https://github.com/sidekiq/sidekiq/discussions/5039
|
218
219
|
EOM
|
219
220
|
RTT_READINGS.reset
|
220
221
|
end
|
@@ -246,14 +247,21 @@ module Sidekiq
|
|
246
247
|
"hostname" => hostname,
|
247
248
|
"started_at" => Time.now.to_f,
|
248
249
|
"pid" => ::Process.pid,
|
249
|
-
"tag" => @
|
250
|
-
"concurrency" => @
|
251
|
-
"queues" => @
|
252
|
-
"
|
253
|
-
"
|
250
|
+
"tag" => @config[:tag] || "",
|
251
|
+
"concurrency" => @config.total_concurrency,
|
252
|
+
"queues" => @config.capsules.values.flat_map { |cap| cap.queues }.uniq,
|
253
|
+
"weights" => to_weights,
|
254
|
+
"labels" => @config[:labels].to_a,
|
255
|
+
"identity" => identity,
|
256
|
+
"version" => Sidekiq::VERSION,
|
257
|
+
"embedded" => @embedded
|
254
258
|
}
|
255
259
|
end
|
256
260
|
|
261
|
+
def to_weights
|
262
|
+
@config.capsules.values.map(&:weights)
|
263
|
+
end
|
264
|
+
|
257
265
|
def to_json
|
258
266
|
# this data changes infrequently so dump it to a string
|
259
267
|
# now so we don't need to dump it every heartbeat.
|
data/lib/sidekiq/logger.rb
CHANGED
@@ -18,7 +18,7 @@ module Sidekiq
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def self.add(k, v)
|
21
|
-
|
21
|
+
current[k] = v
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
@@ -31,28 +31,14 @@ module Sidekiq
|
|
31
31
|
"fatal" => 4
|
32
32
|
}
|
33
33
|
LEVELS.default_proc = proc do |_, level|
|
34
|
-
|
34
|
+
puts("Invalid log level: #{level.inspect}")
|
35
35
|
nil
|
36
36
|
end
|
37
37
|
|
38
|
-
|
39
|
-
level
|
40
|
-
|
41
|
-
|
42
|
-
def info?
|
43
|
-
level <= 1
|
44
|
-
end
|
45
|
-
|
46
|
-
def warn?
|
47
|
-
level <= 2
|
48
|
-
end
|
49
|
-
|
50
|
-
def error?
|
51
|
-
level <= 3
|
52
|
-
end
|
53
|
-
|
54
|
-
def fatal?
|
55
|
-
level <= 4
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
56
42
|
end
|
57
43
|
|
58
44
|
def local_level
|
@@ -84,36 +70,11 @@ module Sidekiq
|
|
84
70
|
ensure
|
85
71
|
self.local_level = old_local_level
|
86
72
|
end
|
87
|
-
|
88
|
-
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
89
|
-
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
90
|
-
def add(severity, message = nil, progname = nil, &block)
|
91
|
-
severity ||= ::Logger::UNKNOWN
|
92
|
-
progname ||= @progname
|
93
|
-
|
94
|
-
return true if @logdev.nil? || severity < level
|
95
|
-
|
96
|
-
if message.nil?
|
97
|
-
if block
|
98
|
-
message = yield
|
99
|
-
else
|
100
|
-
message = progname
|
101
|
-
progname = @progname
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
106
|
-
end
|
107
73
|
end
|
108
74
|
|
109
75
|
class Logger < ::Logger
|
110
76
|
include LoggingUtils
|
111
77
|
|
112
|
-
def initialize(*args, **kwargs)
|
113
|
-
super
|
114
|
-
self.formatter = Sidekiq.log_formatter
|
115
|
-
end
|
116
|
-
|
117
78
|
module Formatters
|
118
79
|
class Base < ::Logger::Formatter
|
119
80
|
def tid
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/processor"
|
5
|
-
require "sidekiq/fetch"
|
6
4
|
require "set"
|
7
5
|
|
8
6
|
module Sidekiq
|
@@ -21,43 +19,38 @@ module Sidekiq
|
|
21
19
|
# the shutdown process. The other tasks are performed by other threads.
|
22
20
|
#
|
23
21
|
class Manager
|
24
|
-
include
|
22
|
+
include Sidekiq::Component
|
25
23
|
|
26
24
|
attr_reader :workers
|
27
|
-
attr_reader :
|
25
|
+
attr_reader :capsule
|
28
26
|
|
29
|
-
def initialize(
|
30
|
-
|
31
|
-
@
|
32
|
-
@count = options[:concurrency] || 10
|
27
|
+
def initialize(capsule)
|
28
|
+
@config = @capsule = capsule
|
29
|
+
@count = capsule.concurrency
|
33
30
|
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
34
31
|
|
35
32
|
@done = false
|
36
33
|
@workers = Set.new
|
34
|
+
@plock = Mutex.new
|
37
35
|
@count.times do
|
38
|
-
@workers << Processor.new(
|
36
|
+
@workers << Processor.new(@config, &method(:processor_result))
|
39
37
|
end
|
40
|
-
@plock = Mutex.new
|
41
38
|
end
|
42
39
|
|
43
40
|
def start
|
44
|
-
@workers.each
|
45
|
-
x.start
|
46
|
-
end
|
41
|
+
@workers.each(&:start)
|
47
42
|
end
|
48
43
|
|
49
44
|
def quiet
|
50
45
|
return if @done
|
51
46
|
@done = true
|
52
47
|
|
53
|
-
logger.info { "Terminating quiet
|
54
|
-
@workers.each
|
55
|
-
fire_event(:quiet, reverse: true)
|
48
|
+
logger.info { "Terminating quiet threads for #{capsule.name} capsule" }
|
49
|
+
@workers.each(&:terminate)
|
56
50
|
end
|
57
51
|
|
58
52
|
def stop(deadline)
|
59
53
|
quiet
|
60
|
-
fire_event(:shutdown, reverse: true)
|
61
54
|
|
62
55
|
# some of the shutdown events can be async,
|
63
56
|
# we don't have any way to know when they're done but
|
@@ -65,24 +58,20 @@ module Sidekiq
|
|
65
58
|
sleep PAUSE_TIME
|
66
59
|
return if @workers.empty?
|
67
60
|
|
68
|
-
logger.info { "Pausing to allow
|
61
|
+
logger.info { "Pausing to allow jobs to finish..." }
|
69
62
|
wait_for(deadline) { @workers.empty? }
|
70
63
|
return if @workers.empty?
|
71
64
|
|
72
65
|
hard_shutdown
|
66
|
+
ensure
|
67
|
+
capsule.stop
|
73
68
|
end
|
74
69
|
|
75
|
-
def
|
76
|
-
@plock.synchronize do
|
77
|
-
@workers.delete(processor)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
def processor_died(processor, reason)
|
70
|
+
def processor_result(processor, reason = nil)
|
82
71
|
@plock.synchronize do
|
83
72
|
@workers.delete(processor)
|
84
73
|
unless @done
|
85
|
-
p = Processor.new(
|
74
|
+
p = Processor.new(@config, &method(:processor_result))
|
86
75
|
@workers << p
|
87
76
|
p.start
|
88
77
|
end
|
@@ -96,7 +85,7 @@ module Sidekiq
|
|
96
85
|
private
|
97
86
|
|
98
87
|
def hard_shutdown
|
99
|
-
# We've reached the timeout and we still have busy
|
88
|
+
# We've reached the timeout and we still have busy threads.
|
100
89
|
# They must die but their jobs shall live on.
|
101
90
|
cleanup = nil
|
102
91
|
@plock.synchronize do
|
@@ -106,17 +95,16 @@ module Sidekiq
|
|
106
95
|
if cleanup.size > 0
|
107
96
|
jobs = cleanup.map { |p| p.job }.compact
|
108
97
|
|
109
|
-
logger.warn { "Terminating #{cleanup.size} busy
|
110
|
-
logger.
|
98
|
+
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
99
|
+
logger.debug { "Jobs still in progress #{jobs.inspect}" }
|
111
100
|
|
112
101
|
# Re-enqueue unfinished jobs
|
113
102
|
# NOTE: You may notice that we may push a job back to redis before
|
114
|
-
# the
|
103
|
+
# the thread is terminated. This is ok because Sidekiq's
|
115
104
|
# contract says that jobs are run AT LEAST once. Process termination
|
116
105
|
# is delayed until we're certain the jobs are back in Redis because
|
117
106
|
# it is worse to lose a job than to run it twice.
|
118
|
-
|
119
|
-
strategy.bulk_requeue(jobs, @options)
|
107
|
+
capsule.fetcher.bulk_requeue(jobs)
|
120
108
|
end
|
121
109
|
|
122
110
|
cleanup.each do |processor|
|
@@ -129,5 +117,18 @@ module Sidekiq
|
|
129
117
|
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
130
118
|
wait_for(deadline) { @workers.empty? }
|
131
119
|
end
|
120
|
+
|
121
|
+
# hack for quicker development / testing environment #2774
|
122
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
123
|
+
|
124
|
+
# Wait for the orblock to be true or the deadline passed.
|
125
|
+
def wait_for(deadline, &condblock)
|
126
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
127
|
+
while remaining > PAUSE_TIME
|
128
|
+
return if condblock.call
|
129
|
+
sleep PAUSE_TIME
|
130
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
131
|
+
end
|
132
|
+
end
|
132
133
|
end
|
133
134
|
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
require "sidekiq"
|
2
|
+
require "date"
|
3
|
+
require "set"
|
4
|
+
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Metrics
|
9
|
+
# Allows caller to query for Sidekiq execution metrics within Redis.
|
10
|
+
# Caller sets a set of attributes to act as filters. {#fetch} will call
|
11
|
+
# Redis and return a Hash of results.
|
12
|
+
#
|
13
|
+
# NB: all metrics and times/dates are UTC only. We specifically do not
|
14
|
+
# support timezones.
|
15
|
+
class Query
|
16
|
+
def initialize(pool: nil, now: Time.now)
|
17
|
+
@time = now.utc
|
18
|
+
@pool = pool || Sidekiq.default_configuration.redis_pool
|
19
|
+
@klass = nil
|
20
|
+
end
|
21
|
+
|
22
|
+
# Get metric data for all jobs from the last hour
|
23
|
+
# +class_filter+: return only results for classes matching filter
|
24
|
+
def top_jobs(class_filter: nil, minutes: 60)
|
25
|
+
result = Result.new
|
26
|
+
|
27
|
+
time = @time
|
28
|
+
redis_results = @pool.with do |conn|
|
29
|
+
conn.pipelined do |pipe|
|
30
|
+
minutes.times do |idx|
|
31
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
32
|
+
pipe.hgetall key
|
33
|
+
result.prepend_bucket time
|
34
|
+
time -= 60
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
time = @time
|
40
|
+
redis_results.each do |hash|
|
41
|
+
hash.each do |k, v|
|
42
|
+
kls, metric = k.split("|")
|
43
|
+
next if class_filter && !class_filter.match?(kls)
|
44
|
+
result.job_results[kls].add_metric metric, time, v.to_i
|
45
|
+
end
|
46
|
+
time -= 60
|
47
|
+
end
|
48
|
+
|
49
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
50
|
+
|
51
|
+
result
|
52
|
+
end
|
53
|
+
|
54
|
+
def for_job(klass, minutes: 60)
|
55
|
+
result = Result.new
|
56
|
+
|
57
|
+
time = @time
|
58
|
+
redis_results = @pool.with do |conn|
|
59
|
+
conn.pipelined do |pipe|
|
60
|
+
minutes.times do |idx|
|
61
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
62
|
+
pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
|
63
|
+
result.prepend_bucket time
|
64
|
+
time -= 60
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
time = @time
|
70
|
+
@pool.with do |conn|
|
71
|
+
redis_results.each do |(ms, p, f)|
|
72
|
+
result.job_results[klass].add_metric "ms", time, ms.to_i if ms
|
73
|
+
result.job_results[klass].add_metric "p", time, p.to_i if p
|
74
|
+
result.job_results[klass].add_metric "f", time, f.to_i if f
|
75
|
+
result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
|
76
|
+
time -= 60
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
81
|
+
|
82
|
+
result
|
83
|
+
end
|
84
|
+
|
85
|
+
class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
|
86
|
+
def initialize
|
87
|
+
super
|
88
|
+
self.buckets = []
|
89
|
+
self.marks = []
|
90
|
+
self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
|
91
|
+
end
|
92
|
+
|
93
|
+
def prepend_bucket(time)
|
94
|
+
buckets.unshift time.strftime("%H:%M")
|
95
|
+
self.ends_at ||= time
|
96
|
+
self.starts_at = time
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
class JobResult < Struct.new(:series, :hist, :totals)
|
101
|
+
def initialize
|
102
|
+
super
|
103
|
+
self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
|
104
|
+
self.hist = Hash.new { |h, k| h[k] = [] }
|
105
|
+
self.totals = Hash.new(0)
|
106
|
+
end
|
107
|
+
|
108
|
+
def add_metric(metric, time, value)
|
109
|
+
totals[metric] += value
|
110
|
+
series[metric][time.strftime("%H:%M")] += value
|
111
|
+
|
112
|
+
# Include timing measurements in seconds for convenience
|
113
|
+
add_metric("s", time, value / 1000.0) if metric == "ms"
|
114
|
+
end
|
115
|
+
|
116
|
+
def add_hist(time, hist_result)
|
117
|
+
hist[time.strftime("%H:%M")] = hist_result
|
118
|
+
end
|
119
|
+
|
120
|
+
def total_avg(metric = "ms")
|
121
|
+
completed = totals["p"] - totals["f"]
|
122
|
+
return 0 if completed.zero?
|
123
|
+
totals[metric].to_f / completed
|
124
|
+
end
|
125
|
+
|
126
|
+
def series_avg(metric = "ms")
|
127
|
+
series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
|
128
|
+
completed = series.dig("p", bucket) - series.dig("f", bucket)
|
129
|
+
result[bucket] = (completed == 0) ? 0 : value.to_f / completed
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class MarkResult < Struct.new(:time, :label)
|
135
|
+
def bucket
|
136
|
+
time.strftime("%H:%M")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
private
|
141
|
+
|
142
|
+
def fetch_marks(time_range)
|
143
|
+
[].tap do |result|
|
144
|
+
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
|
145
|
+
|
146
|
+
marks.each do |timestamp, label|
|
147
|
+
time = Time.parse(timestamp)
|
148
|
+
if time_range.cover? time
|
149
|
+
result << MarkResult.new(time, label)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|