sidekiq 6.3.1 → 7.0.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +205 -11
- data/LICENSE.txt +9 -0
- data/README.md +45 -32
- data/bin/sidekiq +4 -9
- data/bin/sidekiqload +189 -117
- data/bin/sidekiqmon +4 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +308 -188
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +85 -80
- data/lib/sidekiq/client.rb +74 -81
- data/lib/sidekiq/{util.rb → component.rb} +13 -40
- data/lib/sidekiq/config.rb +270 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +23 -24
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +81 -57
- data/lib/sidekiq/job_util.rb +105 -0
- data/lib/sidekiq/launcher.rb +103 -95
- data/lib/sidekiq/logger.rb +9 -44
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +17 -13
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +12 -10
- data/lib/sidekiq/redis_client_adapter.rb +115 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +75 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +45 -11
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +35 -21
- data/lib/sidekiq/web.rb +10 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +85 -202
- data/sidekiq.gemspec +20 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +17 -17
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +69 -302
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +42 -26
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +71 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +68 -32
- data/LICENSE +0 -9
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/delay.rb +0 -41
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/worker.rb +0 -311
data/lib/sidekiq/logger.rb
CHANGED
@@ -16,6 +16,10 @@ module Sidekiq
|
|
16
16
|
def self.current
|
17
17
|
Thread.current[:sidekiq_context] ||= {}
|
18
18
|
end
|
19
|
+
|
20
|
+
def self.add(k, v)
|
21
|
+
current[k] = v
|
22
|
+
end
|
19
23
|
end
|
20
24
|
|
21
25
|
module LoggingUtils
|
@@ -27,28 +31,14 @@ module Sidekiq
|
|
27
31
|
"fatal" => 4
|
28
32
|
}
|
29
33
|
LEVELS.default_proc = proc do |_, level|
|
30
|
-
|
34
|
+
puts("Invalid log level: #{level.inspect}")
|
31
35
|
nil
|
32
36
|
end
|
33
37
|
|
34
|
-
|
35
|
-
level
|
36
|
-
|
37
|
-
|
38
|
-
def info?
|
39
|
-
level <= 1
|
40
|
-
end
|
41
|
-
|
42
|
-
def warn?
|
43
|
-
level <= 2
|
44
|
-
end
|
45
|
-
|
46
|
-
def error?
|
47
|
-
level <= 3
|
48
|
-
end
|
49
|
-
|
50
|
-
def fatal?
|
51
|
-
level <= 4
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
52
42
|
end
|
53
43
|
|
54
44
|
def local_level
|
@@ -80,36 +70,11 @@ module Sidekiq
|
|
80
70
|
ensure
|
81
71
|
self.local_level = old_local_level
|
82
72
|
end
|
83
|
-
|
84
|
-
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
85
|
-
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
86
|
-
def add(severity, message = nil, progname = nil, &block)
|
87
|
-
severity ||= ::Logger::UNKNOWN
|
88
|
-
progname ||= @progname
|
89
|
-
|
90
|
-
return true if @logdev.nil? || severity < level
|
91
|
-
|
92
|
-
if message.nil?
|
93
|
-
if block
|
94
|
-
message = yield
|
95
|
-
else
|
96
|
-
message = progname
|
97
|
-
progname = @progname
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
102
|
-
end
|
103
73
|
end
|
104
74
|
|
105
75
|
class Logger < ::Logger
|
106
76
|
include LoggingUtils
|
107
77
|
|
108
|
-
def initialize(*args, **kwargs)
|
109
|
-
super
|
110
|
-
self.formatter = Sidekiq.log_formatter
|
111
|
-
end
|
112
|
-
|
113
78
|
module Formatters
|
114
79
|
class Base < ::Logger::Formatter
|
115
80
|
def tid
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/processor"
|
5
|
-
require "sidekiq/fetch"
|
6
4
|
require "set"
|
7
5
|
|
8
6
|
module Sidekiq
|
@@ -21,46 +19,38 @@ module Sidekiq
|
|
21
19
|
# the shutdown process. The other tasks are performed by other threads.
|
22
20
|
#
|
23
21
|
class Manager
|
24
|
-
include
|
22
|
+
include Sidekiq::Component
|
25
23
|
|
26
24
|
attr_reader :workers
|
27
|
-
attr_reader :
|
25
|
+
attr_reader :capsule
|
28
26
|
|
29
|
-
def initialize(
|
30
|
-
|
31
|
-
@
|
32
|
-
@count = options[:concurrency] || 10
|
27
|
+
def initialize(capsule)
|
28
|
+
@config = @capsule = capsule
|
29
|
+
@count = capsule.concurrency
|
33
30
|
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
34
31
|
|
35
32
|
@done = false
|
36
33
|
@workers = Set.new
|
34
|
+
@plock = Mutex.new
|
37
35
|
@count.times do
|
38
|
-
@workers << Processor.new(
|
36
|
+
@workers << Processor.new(@config, &method(:processor_result))
|
39
37
|
end
|
40
|
-
@plock = Mutex.new
|
41
38
|
end
|
42
39
|
|
43
40
|
def start
|
44
|
-
@workers.each
|
45
|
-
x.start
|
46
|
-
end
|
41
|
+
@workers.each(&:start)
|
47
42
|
end
|
48
43
|
|
49
44
|
def quiet
|
50
45
|
return if @done
|
51
46
|
@done = true
|
52
47
|
|
53
|
-
logger.info { "Terminating quiet
|
54
|
-
@workers.each
|
55
|
-
fire_event(:quiet, reverse: true)
|
48
|
+
logger.info { "Terminating quiet threads for #{capsule.name} capsule" }
|
49
|
+
@workers.each(&:terminate)
|
56
50
|
end
|
57
51
|
|
58
|
-
# hack for quicker development / testing environment #2774
|
59
|
-
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
60
|
-
|
61
52
|
def stop(deadline)
|
62
53
|
quiet
|
63
|
-
fire_event(:shutdown, reverse: true)
|
64
54
|
|
65
55
|
# some of the shutdown events can be async,
|
66
56
|
# we don't have any way to know when they're done but
|
@@ -68,29 +58,20 @@ module Sidekiq
|
|
68
58
|
sleep PAUSE_TIME
|
69
59
|
return if @workers.empty?
|
70
60
|
|
71
|
-
logger.info { "Pausing to allow
|
72
|
-
|
73
|
-
while remaining > PAUSE_TIME
|
74
|
-
return if @workers.empty?
|
75
|
-
sleep PAUSE_TIME
|
76
|
-
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
77
|
-
end
|
61
|
+
logger.info { "Pausing to allow jobs to finish..." }
|
62
|
+
wait_for(deadline) { @workers.empty? }
|
78
63
|
return if @workers.empty?
|
79
64
|
|
80
65
|
hard_shutdown
|
66
|
+
ensure
|
67
|
+
capsule.stop
|
81
68
|
end
|
82
69
|
|
83
|
-
def
|
84
|
-
@plock.synchronize do
|
85
|
-
@workers.delete(processor)
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def processor_died(processor, reason)
|
70
|
+
def processor_result(processor, reason = nil)
|
90
71
|
@plock.synchronize do
|
91
72
|
@workers.delete(processor)
|
92
73
|
unless @done
|
93
|
-
p = Processor.new(
|
74
|
+
p = Processor.new(@config, &method(:processor_result))
|
94
75
|
@workers << p
|
95
76
|
p.start
|
96
77
|
end
|
@@ -104,7 +85,7 @@ module Sidekiq
|
|
104
85
|
private
|
105
86
|
|
106
87
|
def hard_shutdown
|
107
|
-
# We've reached the timeout and we still have busy
|
88
|
+
# We've reached the timeout and we still have busy threads.
|
108
89
|
# They must die but their jobs shall live on.
|
109
90
|
cleanup = nil
|
110
91
|
@plock.synchronize do
|
@@ -114,22 +95,40 @@ module Sidekiq
|
|
114
95
|
if cleanup.size > 0
|
115
96
|
jobs = cleanup.map { |p| p.job }.compact
|
116
97
|
|
117
|
-
logger.warn { "Terminating #{cleanup.size} busy
|
118
|
-
logger.
|
98
|
+
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
99
|
+
logger.debug { "Jobs still in progress #{jobs.inspect}" }
|
119
100
|
|
120
101
|
# Re-enqueue unfinished jobs
|
121
102
|
# NOTE: You may notice that we may push a job back to redis before
|
122
|
-
# the
|
103
|
+
# the thread is terminated. This is ok because Sidekiq's
|
123
104
|
# contract says that jobs are run AT LEAST once. Process termination
|
124
105
|
# is delayed until we're certain the jobs are back in Redis because
|
125
106
|
# it is worse to lose a job than to run it twice.
|
126
|
-
|
127
|
-
strategy.bulk_requeue(jobs, @options)
|
107
|
+
capsule.fetcher.bulk_requeue(jobs)
|
128
108
|
end
|
129
109
|
|
130
110
|
cleanup.each do |processor|
|
131
111
|
processor.kill
|
132
112
|
end
|
113
|
+
|
114
|
+
# when this method returns, we immediately call `exit` which may not give
|
115
|
+
# the remaining threads time to run `ensure` blocks, etc. We pause here up
|
116
|
+
# to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
|
117
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
118
|
+
wait_for(deadline) { @workers.empty? }
|
119
|
+
end
|
120
|
+
|
121
|
+
# hack for quicker development / testing environment #2774
|
122
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
123
|
+
|
124
|
+
# Wait for the orblock to be true or the deadline passed.
|
125
|
+
def wait_for(deadline, &condblock)
|
126
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
127
|
+
while remaining > PAUSE_TIME
|
128
|
+
return if condblock.call
|
129
|
+
sleep PAUSE_TIME
|
130
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
131
|
+
end
|
133
132
|
end
|
134
133
|
end
|
135
134
|
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
require "sidekiq"
|
2
|
+
require "date"
|
3
|
+
require "set"
|
4
|
+
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Metrics
|
9
|
+
# Allows caller to query for Sidekiq execution metrics within Redis.
|
10
|
+
# Caller sets a set of attributes to act as filters. {#fetch} will call
|
11
|
+
# Redis and return a Hash of results.
|
12
|
+
#
|
13
|
+
# NB: all metrics and times/dates are UTC only. We specifically do not
|
14
|
+
# support timezones.
|
15
|
+
class Query
|
16
|
+
def initialize(pool: nil, now: Time.now)
|
17
|
+
@time = now.utc
|
18
|
+
@pool = pool || Sidekiq.default_configuration.redis_pool
|
19
|
+
@klass = nil
|
20
|
+
end
|
21
|
+
|
22
|
+
# Get metric data for all jobs from the last hour
|
23
|
+
def top_jobs(minutes: 60)
|
24
|
+
result = Result.new
|
25
|
+
|
26
|
+
time = @time
|
27
|
+
redis_results = @pool.with do |conn|
|
28
|
+
conn.pipelined do |pipe|
|
29
|
+
minutes.times do |idx|
|
30
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
31
|
+
pipe.hgetall key
|
32
|
+
result.prepend_bucket time
|
33
|
+
time -= 60
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
time = @time
|
39
|
+
redis_results.each do |hash|
|
40
|
+
hash.each do |k, v|
|
41
|
+
kls, metric = k.split("|")
|
42
|
+
result.job_results[kls].add_metric metric, time, v.to_i
|
43
|
+
end
|
44
|
+
time -= 60
|
45
|
+
end
|
46
|
+
|
47
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
48
|
+
|
49
|
+
result
|
50
|
+
end
|
51
|
+
|
52
|
+
def for_job(klass, minutes: 60)
|
53
|
+
result = Result.new
|
54
|
+
|
55
|
+
time = @time
|
56
|
+
redis_results = @pool.with do |conn|
|
57
|
+
conn.pipelined do |pipe|
|
58
|
+
minutes.times do |idx|
|
59
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
60
|
+
pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
|
61
|
+
result.prepend_bucket time
|
62
|
+
time -= 60
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
time = @time
|
68
|
+
@pool.with do |conn|
|
69
|
+
redis_results.each do |(ms, p, f)|
|
70
|
+
result.job_results[klass].add_metric "ms", time, ms.to_i if ms
|
71
|
+
result.job_results[klass].add_metric "p", time, p.to_i if p
|
72
|
+
result.job_results[klass].add_metric "f", time, f.to_i if f
|
73
|
+
result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time)
|
74
|
+
time -= 60
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
79
|
+
|
80
|
+
result
|
81
|
+
end
|
82
|
+
|
83
|
+
class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
|
84
|
+
def initialize
|
85
|
+
super
|
86
|
+
self.buckets = []
|
87
|
+
self.marks = []
|
88
|
+
self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
|
89
|
+
end
|
90
|
+
|
91
|
+
def prepend_bucket(time)
|
92
|
+
buckets.unshift time.strftime("%H:%M")
|
93
|
+
self.ends_at ||= time
|
94
|
+
self.starts_at = time
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class JobResult < Struct.new(:series, :hist, :totals)
|
99
|
+
def initialize
|
100
|
+
super
|
101
|
+
self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
|
102
|
+
self.hist = Hash.new { |h, k| h[k] = [] }
|
103
|
+
self.totals = Hash.new(0)
|
104
|
+
end
|
105
|
+
|
106
|
+
def add_metric(metric, time, value)
|
107
|
+
totals[metric] += value
|
108
|
+
series[metric][time.strftime("%H:%M")] += value
|
109
|
+
|
110
|
+
# Include timing measurements in seconds for convenience
|
111
|
+
add_metric("s", time, value / 1000.0) if metric == "ms"
|
112
|
+
end
|
113
|
+
|
114
|
+
def add_hist(time, hist_result)
|
115
|
+
hist[time.strftime("%H:%M")] = hist_result
|
116
|
+
end
|
117
|
+
|
118
|
+
def total_avg(metric = "ms")
|
119
|
+
completed = totals["p"] - totals["f"]
|
120
|
+
totals[metric].to_f / completed
|
121
|
+
end
|
122
|
+
|
123
|
+
def series_avg(metric = "ms")
|
124
|
+
series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
|
125
|
+
completed = series.dig("p", bucket) - series.dig("f", bucket)
|
126
|
+
result[bucket] = (completed == 0) ? 0 : value.to_f / completed
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
class MarkResult < Struct.new(:time, :label)
|
132
|
+
def bucket
|
133
|
+
time.strftime("%H:%M")
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
private
|
138
|
+
|
139
|
+
def fetch_marks(time_range)
|
140
|
+
[].tap do |result|
|
141
|
+
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
|
142
|
+
|
143
|
+
marks.each do |timestamp, label|
|
144
|
+
time = Time.parse(timestamp)
|
145
|
+
if time_range.cover? time
|
146
|
+
result << MarkResult.new(time, label)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Metrics
|
5
|
+
# This is the only dependency on concurrent-ruby in Sidekiq but it's
|
6
|
+
# mandatory for thread-safety until MRI supports atomic operations on values.
|
7
|
+
Counter = ::Concurrent::AtomicFixnum
|
8
|
+
|
9
|
+
# Implements space-efficient but statistically useful histogram storage.
|
10
|
+
# A precise time histogram stores every time. Instead we break times into a set of
|
11
|
+
# known buckets and increment counts of the associated time bucket. Even if we call
|
12
|
+
# the histogram a million times, we'll still only store 26 buckets.
|
13
|
+
# NB: needs to be thread-safe or resiliant to races.
|
14
|
+
#
|
15
|
+
# To store this data, we use Redis' BITFIELD command to store unsigned 16-bit counters
|
16
|
+
# per bucket per klass per minute. It's unlikely that most people will be executing more
|
17
|
+
# than 1000 job/sec for a full minute of a specific type.
|
18
|
+
class Histogram
|
19
|
+
include Enumerable
|
20
|
+
|
21
|
+
# This number represents the maximum milliseconds for this bucket.
|
22
|
+
# 20 means all job executions up to 20ms, e.g. if a job takes
|
23
|
+
# 280ms, it'll increment bucket[7]. Note we can track job executions
|
24
|
+
# up to about 5.5 minutes. After that, it's assumed you're probably
|
25
|
+
# not too concerned with its performance.
|
26
|
+
BUCKET_INTERVALS = [
|
27
|
+
20, 30, 45, 65, 100,
|
28
|
+
150, 225, 335, 500, 750,
|
29
|
+
1100, 1700, 2500, 3800, 5750,
|
30
|
+
8500, 13000, 20000, 30000, 45000,
|
31
|
+
65000, 100000, 150000, 225000, 335000,
|
32
|
+
Float::INFINITY # the "maybe your job is too long" bucket
|
33
|
+
]
|
34
|
+
LABELS = [
|
35
|
+
"20ms", "30ms", "45ms", "65ms", "100ms",
|
36
|
+
"150ms", "225ms", "335ms", "500ms", "750ms",
|
37
|
+
"1.1s", "1.7s", "2.5s", "3.8s", "5.75s",
|
38
|
+
"8.5s", "13s", "20s", "30s", "45s",
|
39
|
+
"65s", "100s", "150s", "225s", "335s",
|
40
|
+
"Slow"
|
41
|
+
]
|
42
|
+
FETCH = "GET u16 #0 GET u16 #1 GET u16 #2 GET u16 #3 \
|
43
|
+
GET u16 #4 GET u16 #5 GET u16 #6 GET u16 #7 \
|
44
|
+
GET u16 #8 GET u16 #9 GET u16 #10 GET u16 #11 \
|
45
|
+
GET u16 #12 GET u16 #13 GET u16 #14 GET u16 #15 \
|
46
|
+
GET u16 #16 GET u16 #17 GET u16 #18 GET u16 #19 \
|
47
|
+
GET u16 #20 GET u16 #21 GET u16 #22 GET u16 #23 \
|
48
|
+
GET u16 #24 GET u16 #25".split
|
49
|
+
HISTOGRAM_TTL = 8 * 60 * 60
|
50
|
+
|
51
|
+
def each
|
52
|
+
buckets.each { |counter| yield counter.value }
|
53
|
+
end
|
54
|
+
|
55
|
+
def label(idx)
|
56
|
+
LABELS[idx]
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_reader :buckets
|
60
|
+
def initialize(klass)
|
61
|
+
@klass = klass
|
62
|
+
@buckets = Array.new(BUCKET_INTERVALS.size) { Counter.new }
|
63
|
+
end
|
64
|
+
|
65
|
+
def record_time(ms)
|
66
|
+
index_to_use = BUCKET_INTERVALS.each_index do |idx|
|
67
|
+
break idx if ms < BUCKET_INTERVALS[idx]
|
68
|
+
end
|
69
|
+
|
70
|
+
@buckets[index_to_use].increment
|
71
|
+
end
|
72
|
+
|
73
|
+
def fetch(conn, now = Time.now)
|
74
|
+
window = now.utc.strftime("%d-%H:%-M")
|
75
|
+
key = "#{@klass}-#{window}"
|
76
|
+
conn.bitfield(key, *FETCH)
|
77
|
+
end
|
78
|
+
|
79
|
+
def persist(conn, now = Time.now)
|
80
|
+
buckets, @buckets = @buckets, []
|
81
|
+
window = now.utc.strftime("%d-%H:%-M")
|
82
|
+
key = "#{@klass}-#{window}"
|
83
|
+
cmd = [key, "OVERFLOW", "SAT"]
|
84
|
+
buckets.each_with_index do |counter, idx|
|
85
|
+
val = counter.value
|
86
|
+
cmd << "INCRBY" << "u16" << "##{idx}" << val.to_s if val > 0
|
87
|
+
end
|
88
|
+
|
89
|
+
conn.bitfield(*cmd) if cmd.size > 3
|
90
|
+
conn.expire(key, HISTOGRAM_TTL)
|
91
|
+
key
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "time"
|
4
|
+
require "sidekiq"
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
# This file contains the components which track execution metrics within Sidekiq.
|
8
|
+
module Sidekiq
|
9
|
+
module Metrics
|
10
|
+
class ExecutionTracker
|
11
|
+
include Sidekiq::Component
|
12
|
+
|
13
|
+
def initialize(config)
|
14
|
+
@config = config
|
15
|
+
@jobs = Hash.new(0)
|
16
|
+
@totals = Hash.new(0)
|
17
|
+
@grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
|
18
|
+
@lock = Mutex.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def track(queue, klass)
|
22
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
23
|
+
time_ms = 0
|
24
|
+
begin
|
25
|
+
begin
|
26
|
+
yield
|
27
|
+
ensure
|
28
|
+
finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
29
|
+
time_ms = finish - start
|
30
|
+
end
|
31
|
+
# We don't track time for failed jobs as they can have very unpredictable
|
32
|
+
# execution times. more important to know average time for successful jobs so we
|
33
|
+
# can better recognize when a perf regression is introduced.
|
34
|
+
@lock.synchronize {
|
35
|
+
@grams[klass].record_time(time_ms)
|
36
|
+
@jobs["#{klass}|ms"] += time_ms
|
37
|
+
@totals["ms"] += time_ms
|
38
|
+
}
|
39
|
+
rescue Exception
|
40
|
+
@lock.synchronize {
|
41
|
+
@jobs["#{klass}|f"] += 1
|
42
|
+
@totals["f"] += 1
|
43
|
+
}
|
44
|
+
raise
|
45
|
+
ensure
|
46
|
+
@lock.synchronize {
|
47
|
+
@jobs["#{klass}|p"] += 1
|
48
|
+
@totals["p"] += 1
|
49
|
+
}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# LONG_TERM = 90 * 24 * 60 * 60
|
54
|
+
# MID_TERM = 7 * 24 * 60 * 60
|
55
|
+
SHORT_TERM = 8 * 60 * 60
|
56
|
+
|
57
|
+
def flush(time = Time.now)
|
58
|
+
totals, jobs, grams = reset
|
59
|
+
procd = totals["p"]
|
60
|
+
fails = totals["f"]
|
61
|
+
return if procd == 0 && fails == 0
|
62
|
+
|
63
|
+
now = time.utc
|
64
|
+
# nowdate = now.strftime("%Y%m%d")
|
65
|
+
# nowhour = now.strftime("%Y%m%d|%-H")
|
66
|
+
nowmin = now.strftime("%Y%m%d|%-H:%-M")
|
67
|
+
count = 0
|
68
|
+
|
69
|
+
redis do |conn|
|
70
|
+
# persist fine-grained histogram data
|
71
|
+
if grams.size > 0
|
72
|
+
conn.pipelined do |pipe|
|
73
|
+
grams.each do |_, gram|
|
74
|
+
gram.persist(pipe, now)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# persist coarse grained execution count + execution millis.
|
80
|
+
# note as of today we don't use or do anything with the
|
81
|
+
# daily or hourly rollups.
|
82
|
+
[
|
83
|
+
# ["j", jobs, nowdate, LONG_TERM],
|
84
|
+
# ["j", jobs, nowhour, MID_TERM],
|
85
|
+
["j", jobs, nowmin, SHORT_TERM]
|
86
|
+
].each do |prefix, data, bucket, ttl|
|
87
|
+
conn.pipelined do |xa|
|
88
|
+
stats = "#{prefix}|#{bucket}"
|
89
|
+
data.each_pair do |key, value|
|
90
|
+
xa.hincrby stats, key, value
|
91
|
+
count += 1
|
92
|
+
end
|
93
|
+
xa.expire(stats, ttl)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
logger.debug "Flushed #{count} metrics"
|
97
|
+
count
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
def reset
|
104
|
+
@lock.synchronize {
|
105
|
+
array = [@totals, @jobs, @grams]
|
106
|
+
@totals = Hash.new(0)
|
107
|
+
@jobs = Hash.new(0)
|
108
|
+
@grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
|
109
|
+
array
|
110
|
+
}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class Middleware
|
115
|
+
include Sidekiq::ServerMiddleware
|
116
|
+
|
117
|
+
def initialize(options)
|
118
|
+
@exec = options
|
119
|
+
end
|
120
|
+
|
121
|
+
def call(_instance, hash, queue, &block)
|
122
|
+
@exec.track(queue, hash["wrapped"] || hash["class"], &block)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
Sidekiq.configure_server do |config|
|
129
|
+
exec = Sidekiq::Metrics::ExecutionTracker.new(config)
|
130
|
+
config.server_middleware do |chain|
|
131
|
+
chain.add Sidekiq::Metrics::Middleware, exec
|
132
|
+
end
|
133
|
+
config.on(:beat) do
|
134
|
+
exec.flush
|
135
|
+
end
|
136
|
+
end
|