sidekiq 3.5.4 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +992 -6
- data/LICENSE.txt +9 -0
- data/README.md +52 -43
- data/bin/sidekiq +22 -4
- data/bin/sidekiqload +209 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +633 -295
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +270 -248
- data/lib/sidekiq/client.rb +139 -108
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +53 -121
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +301 -0
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +241 -69
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +88 -190
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +114 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +8 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +29 -16
- data/lib/sidekiq/processor.rb +238 -118
- data/lib/sidekiq/rails.rb +57 -27
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +49 -50
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +173 -52
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +7 -5
- data/lib/sidekiq/testing.rb +197 -65
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +4 -1
- data/lib/sidekiq/web/action.rb +93 -0
- data/lib/sidekiq/web/application.rb +463 -0
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +364 -0
- data/lib/sidekiq/web/router.rb +104 -0
- data/lib/sidekiq/web.rb +113 -216
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +99 -142
- data/sidekiq.gemspec +26 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +163 -74
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +37 -280
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +153 -0
- data/web/assets/stylesheets/application.css +181 -198
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +4 -8
- data/web/locales/ar.yml +87 -0
- data/web/locales/cs.yml +62 -52
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +86 -62
- data/web/locales/es.yml +70 -53
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +86 -56
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +80 -0
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +78 -56
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -60
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +10 -9
- data/web/views/_job_info.erb +26 -5
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +6 -20
- data/web/views/_paging.erb +3 -1
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +87 -28
- data/web/views/dashboard.erb +51 -21
- data/web/views/dead.erb +4 -4
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +15 -5
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +25 -22
- data/web/views/queue.erb +35 -25
- data/web/views/queues.erb +23 -7
- data/web/views/retries.erb +28 -23
- data/web/views/retry.erb +5 -5
- data/web/views/scheduled.erb +19 -17
- data/web/views/scheduled_job_info.erb +1 -1
- metadata +86 -268
- data/.gitignore +0 -12
- data/.travis.yml +0 -16
- data/3.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -95
- data/Contributing.md +0 -32
- data/Ent-Changes.md +0 -39
- data/Gemfile +0 -27
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-Changes.md +0 -454
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -93
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/actor.rb +0 -39
- data/lib/sidekiq/core_ext.rb +0 -105
- data/lib/sidekiq/exception_handler.rb +0 -30
- data/lib/sidekiq/extensions/action_mailer.rb +0 -56
- data/lib/sidekiq/extensions/active_record.rb +0 -39
- data/lib/sidekiq/extensions/class_methods.rb +0 -39
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -24
- data/lib/sidekiq/logging.rb +0 -104
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -206
- data/lib/sidekiq/util.rb +0 -68
- data/lib/sidekiq/web_helpers.rb +0 -249
- data/lib/sidekiq/worker.rb +0 -103
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -0
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -49
- data/test/test_api.rb +0 -493
- data/test/test_cli.rb +0 -335
- data/test/test_client.rb +0 -194
- data/test/test_exception_handler.rb +0 -55
- data/test/test_extensions.rb +0 -126
- data/test/test_fetch.rb +0 -104
- data/test/test_logging.rb +0 -34
- data/test/test_manager.rb +0 -168
- data/test/test_middleware.rb +0 -159
- data/test/test_processor.rb +0 -237
- data/test/test_rails.rb +0 -21
- data/test/test_redis_connection.rb +0 -126
- data/test/test_retry.rb +0 -325
- data/test/test_scheduled.rb +0 -114
- data/test/test_scheduling.rb +0 -49
- data/test/test_sidekiq.rb +0 -99
- data/test/test_testing.rb +0 -142
- data/test/test_testing_fake.rb +0 -268
- data/test/test_testing_inline.rb +0 -93
- data/test/test_util.rb +0 -16
- data/test/test_web.rb +0 -608
- data/test/test_web_helpers.rb +0 -53
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/javascripts/locales/README.md +0 -27
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +0 -96
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.da.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.de.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.el.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.en.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.es.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.et.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +0 -22
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +0 -28
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +0 -17
- data/web/assets/javascripts/locales/jquery.timeago.he.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.id.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.it.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +0 -19
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +0 -17
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.no.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +0 -31
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +0 -34
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +0 -44
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.th.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +0 -34
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +0 -19
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +0 -20
- data/web/views/_poll_js.erb +0 -5
- /data/web/assets/images/{status-sd8051fd480.png → status.png} +0 -0
@@ -0,0 +1,136 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "time"
|
4
|
+
require "sidekiq"
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
# This file contains the components which track execution metrics within Sidekiq.
|
8
|
+
module Sidekiq
|
9
|
+
module Metrics
|
10
|
+
class ExecutionTracker
|
11
|
+
include Sidekiq::Component
|
12
|
+
|
13
|
+
def initialize(config)
|
14
|
+
@config = config
|
15
|
+
@jobs = Hash.new(0)
|
16
|
+
@totals = Hash.new(0)
|
17
|
+
@grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
|
18
|
+
@lock = Mutex.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def track(queue, klass)
|
22
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
23
|
+
time_ms = 0
|
24
|
+
begin
|
25
|
+
begin
|
26
|
+
yield
|
27
|
+
ensure
|
28
|
+
finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
29
|
+
time_ms = finish - start
|
30
|
+
end
|
31
|
+
# We don't track time for failed jobs as they can have very unpredictable
|
32
|
+
# execution times. more important to know average time for successful jobs so we
|
33
|
+
# can better recognize when a perf regression is introduced.
|
34
|
+
@lock.synchronize {
|
35
|
+
@grams[klass].record_time(time_ms)
|
36
|
+
@jobs["#{klass}|ms"] += time_ms
|
37
|
+
@totals["ms"] += time_ms
|
38
|
+
}
|
39
|
+
rescue Exception
|
40
|
+
@lock.synchronize {
|
41
|
+
@jobs["#{klass}|f"] += 1
|
42
|
+
@totals["f"] += 1
|
43
|
+
}
|
44
|
+
raise
|
45
|
+
ensure
|
46
|
+
@lock.synchronize {
|
47
|
+
@jobs["#{klass}|p"] += 1
|
48
|
+
@totals["p"] += 1
|
49
|
+
}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# LONG_TERM = 90 * 24 * 60 * 60
|
54
|
+
# MID_TERM = 7 * 24 * 60 * 60
|
55
|
+
SHORT_TERM = 8 * 60 * 60
|
56
|
+
|
57
|
+
def flush(time = Time.now)
|
58
|
+
totals, jobs, grams = reset
|
59
|
+
procd = totals["p"]
|
60
|
+
fails = totals["f"]
|
61
|
+
return if procd == 0 && fails == 0
|
62
|
+
|
63
|
+
now = time.utc
|
64
|
+
# nowdate = now.strftime("%Y%m%d")
|
65
|
+
# nowhour = now.strftime("%Y%m%d|%-H")
|
66
|
+
nowmin = now.strftime("%Y%m%d|%-H:%-M")
|
67
|
+
count = 0
|
68
|
+
|
69
|
+
redis do |conn|
|
70
|
+
# persist fine-grained histogram data
|
71
|
+
if grams.size > 0
|
72
|
+
conn.pipelined do |pipe|
|
73
|
+
grams.each do |_, gram|
|
74
|
+
gram.persist(pipe, now)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# persist coarse grained execution count + execution millis.
|
80
|
+
# note as of today we don't use or do anything with the
|
81
|
+
# daily or hourly rollups.
|
82
|
+
[
|
83
|
+
# ["j", jobs, nowdate, LONG_TERM],
|
84
|
+
# ["j", jobs, nowhour, MID_TERM],
|
85
|
+
["j", jobs, nowmin, SHORT_TERM]
|
86
|
+
].each do |prefix, data, bucket, ttl|
|
87
|
+
conn.pipelined do |xa|
|
88
|
+
stats = "#{prefix}|#{bucket}"
|
89
|
+
data.each_pair do |key, value|
|
90
|
+
xa.hincrby stats, key, value
|
91
|
+
count += 1
|
92
|
+
end
|
93
|
+
xa.expire(stats, ttl)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
logger.debug "Flushed #{count} metrics"
|
97
|
+
count
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
def reset
|
104
|
+
@lock.synchronize {
|
105
|
+
array = [@totals, @jobs, @grams]
|
106
|
+
@totals = Hash.new(0)
|
107
|
+
@jobs = Hash.new(0)
|
108
|
+
@grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
|
109
|
+
array
|
110
|
+
}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class Middleware
|
115
|
+
include Sidekiq::ServerMiddleware
|
116
|
+
|
117
|
+
def initialize(options)
|
118
|
+
@exec = options
|
119
|
+
end
|
120
|
+
|
121
|
+
def call(_instance, hash, queue, &block)
|
122
|
+
@exec.track(queue, hash["wrapped"] || hash["class"], &block)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
Sidekiq.configure_server do |config|
|
129
|
+
exec = Sidekiq::Metrics::ExecutionTracker.new(config)
|
130
|
+
config.server_middleware do |chain|
|
131
|
+
chain.add Sidekiq::Metrics::Middleware, exec
|
132
|
+
end
|
133
|
+
config.on(:beat) do
|
134
|
+
exec.flush
|
135
|
+
end
|
136
|
+
end
|
@@ -1,116 +1,160 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq/middleware/modules"
|
4
|
+
|
1
5
|
module Sidekiq
|
2
6
|
# Middleware is code configured to run before/after
|
3
|
-
# a
|
7
|
+
# a job is processed. It is patterned after Rack
|
4
8
|
# middleware. Middleware exists for the client side
|
5
9
|
# (pushing jobs onto the queue) as well as the server
|
6
10
|
# side (when jobs are actually processed).
|
7
11
|
#
|
12
|
+
# Callers will register middleware Classes and Sidekiq will
|
13
|
+
# create new instances of the middleware for every job. This
|
14
|
+
# is important so that instance state is not shared accidentally
|
15
|
+
# between job executions.
|
16
|
+
#
|
8
17
|
# To add middleware for the client:
|
9
18
|
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
#
|
19
|
+
# Sidekiq.configure_client do |config|
|
20
|
+
# config.client_middleware do |chain|
|
21
|
+
# chain.add MyClientHook
|
22
|
+
# end
|
13
23
|
# end
|
14
|
-
# end
|
15
24
|
#
|
16
25
|
# To modify middleware for the server, just call
|
17
26
|
# with another block:
|
18
27
|
#
|
19
|
-
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
28
|
+
# Sidekiq.configure_server do |config|
|
29
|
+
# config.server_middleware do |chain|
|
30
|
+
# chain.add MyServerHook
|
31
|
+
# chain.remove ActiveRecord
|
32
|
+
# end
|
23
33
|
# end
|
24
|
-
# end
|
25
34
|
#
|
26
35
|
# To insert immediately preceding another entry:
|
27
36
|
#
|
28
|
-
#
|
29
|
-
#
|
30
|
-
#
|
37
|
+
# Sidekiq.configure_client do |config|
|
38
|
+
# config.client_middleware do |chain|
|
39
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
40
|
+
# end
|
31
41
|
# end
|
32
|
-
# end
|
33
42
|
#
|
34
43
|
# To insert immediately after another entry:
|
35
44
|
#
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
45
|
+
# Sidekiq.configure_client do |config|
|
46
|
+
# config.client_middleware do |chain|
|
47
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
48
|
+
# end
|
39
49
|
# end
|
40
|
-
# end
|
41
50
|
#
|
42
51
|
# This is an example of a minimal server middleware:
|
43
52
|
#
|
44
|
-
#
|
45
|
-
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
#
|
53
|
+
# class MyServerHook
|
54
|
+
# include Sidekiq::ServerMiddleware
|
55
|
+
#
|
56
|
+
# def call(job_instance, msg, queue)
|
57
|
+
# logger.info "Before job"
|
58
|
+
# redis {|conn| conn.get("foo") } # do something in Redis
|
59
|
+
# yield
|
60
|
+
# logger.info "After job"
|
61
|
+
# end
|
49
62
|
# end
|
50
|
-
# end
|
51
63
|
#
|
52
64
|
# This is an example of a minimal client middleware, note
|
53
65
|
# the method must return the result or the job will not push
|
54
66
|
# to Redis:
|
55
67
|
#
|
56
|
-
#
|
57
|
-
#
|
58
|
-
#
|
59
|
-
#
|
60
|
-
#
|
61
|
-
#
|
68
|
+
# class MyClientHook
|
69
|
+
# include Sidekiq::ClientMiddleware
|
70
|
+
#
|
71
|
+
# def call(job_class, msg, queue, redis_pool)
|
72
|
+
# logger.info "Before push"
|
73
|
+
# result = yield
|
74
|
+
# logger.info "After push"
|
75
|
+
# result
|
76
|
+
# end
|
62
77
|
# end
|
63
|
-
# end
|
64
78
|
#
|
65
79
|
module Middleware
|
66
80
|
class Chain
|
67
81
|
include Enumerable
|
68
|
-
attr_reader :entries
|
69
|
-
|
70
|
-
def initialize_copy(copy)
|
71
|
-
copy.instance_variable_set(:@entries, entries.dup)
|
72
|
-
end
|
73
82
|
|
83
|
+
# Iterate through each middleware in the chain
|
74
84
|
def each(&block)
|
75
85
|
entries.each(&block)
|
76
86
|
end
|
77
87
|
|
78
|
-
|
79
|
-
|
88
|
+
# @api private
|
89
|
+
def initialize(config = nil) # :nodoc:
|
90
|
+
@config = config
|
91
|
+
@entries = nil
|
80
92
|
yield self if block_given?
|
81
93
|
end
|
82
94
|
|
95
|
+
def entries
|
96
|
+
@entries ||= []
|
97
|
+
end
|
98
|
+
|
99
|
+
def copy_for(capsule)
|
100
|
+
chain = Sidekiq::Middleware::Chain.new(capsule)
|
101
|
+
chain.instance_variable_set(:@entries, entries.dup)
|
102
|
+
chain
|
103
|
+
end
|
104
|
+
|
105
|
+
# Remove all middleware matching the given Class
|
106
|
+
# @param klass [Class]
|
83
107
|
def remove(klass)
|
84
108
|
entries.delete_if { |entry| entry.klass == klass }
|
85
109
|
end
|
86
110
|
|
111
|
+
# Add the given middleware to the end of the chain.
|
112
|
+
# Sidekiq will call `klass.new(*args)` to create a clean
|
113
|
+
# copy of your middleware for every job executed.
|
114
|
+
#
|
115
|
+
# chain.add(Statsd::Metrics, { collector: "localhost:8125" })
|
116
|
+
#
|
117
|
+
# @param klass [Class] Your middleware class
|
118
|
+
# @param *args [Array<Object>] Set of arguments to pass to every instance of your middleware
|
87
119
|
def add(klass, *args)
|
88
|
-
remove(klass)
|
89
|
-
entries << Entry.new(klass, *args)
|
120
|
+
remove(klass)
|
121
|
+
entries << Entry.new(@config, klass, *args)
|
90
122
|
end
|
91
123
|
|
124
|
+
# Identical to {#add} except the middleware is added to the front of the chain.
|
92
125
|
def prepend(klass, *args)
|
93
|
-
remove(klass)
|
94
|
-
entries.insert(0, Entry.new(klass, *args))
|
126
|
+
remove(klass)
|
127
|
+
entries.insert(0, Entry.new(@config, klass, *args))
|
95
128
|
end
|
96
129
|
|
130
|
+
# Inserts +newklass+ before +oldklass+ in the chain.
|
131
|
+
# Useful if one middleware must run before another middleware.
|
97
132
|
def insert_before(oldklass, newklass, *args)
|
98
133
|
i = entries.index { |entry| entry.klass == newklass }
|
99
|
-
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
134
|
+
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
100
135
|
i = entries.index { |entry| entry.klass == oldklass } || 0
|
101
136
|
entries.insert(i, new_entry)
|
102
137
|
end
|
103
138
|
|
139
|
+
# Inserts +newklass+ after +oldklass+ in the chain.
|
140
|
+
# Useful if one middleware must run after another middleware.
|
104
141
|
def insert_after(oldklass, newklass, *args)
|
105
142
|
i = entries.index { |entry| entry.klass == newklass }
|
106
|
-
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
143
|
+
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
107
144
|
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
108
|
-
entries.insert(i+1, new_entry)
|
145
|
+
entries.insert(i + 1, new_entry)
|
109
146
|
end
|
110
147
|
|
148
|
+
# @return [Boolean] if the given class is already in the chain
|
111
149
|
def exists?(klass)
|
112
150
|
any? { |entry| entry.klass == klass }
|
113
151
|
end
|
152
|
+
alias_method :include?, :exists?
|
153
|
+
|
154
|
+
# @return [Boolean] if the chain contains no middleware
|
155
|
+
def empty?
|
156
|
+
@entries.nil? || @entries.empty?
|
157
|
+
end
|
114
158
|
|
115
159
|
def retrieve
|
116
160
|
map(&:make_new)
|
@@ -120,29 +164,43 @@ module Sidekiq
|
|
120
164
|
entries.clear
|
121
165
|
end
|
122
166
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
167
|
+
# Used by Sidekiq to execute the middleware at runtime
|
168
|
+
# @api private
|
169
|
+
def invoke(*args, &block)
|
170
|
+
return yield if empty?
|
171
|
+
|
172
|
+
chain = retrieve
|
173
|
+
traverse(chain, 0, args, &block)
|
174
|
+
end
|
175
|
+
|
176
|
+
private
|
177
|
+
|
178
|
+
def traverse(chain, index, args, &block)
|
179
|
+
if index >= chain.size
|
180
|
+
yield
|
181
|
+
else
|
182
|
+
chain[index].call(*args) do
|
183
|
+
traverse(chain, index + 1, args, &block)
|
130
184
|
end
|
131
185
|
end
|
132
|
-
traverse_chain.call
|
133
186
|
end
|
134
187
|
end
|
135
188
|
|
189
|
+
# Represents each link in the middleware chain
|
190
|
+
# @api private
|
136
191
|
class Entry
|
137
192
|
attr_reader :klass
|
138
193
|
|
139
|
-
def initialize(klass, *args)
|
194
|
+
def initialize(config, klass, *args)
|
195
|
+
@config = config
|
140
196
|
@klass = klass
|
141
|
-
@args
|
197
|
+
@args = args
|
142
198
|
end
|
143
199
|
|
144
200
|
def make_new
|
145
|
-
@klass.new(*@args)
|
201
|
+
x = @klass.new(*@args)
|
202
|
+
x.config = @config if @config && x.respond_to?(:config=)
|
203
|
+
x
|
146
204
|
end
|
147
205
|
end
|
148
206
|
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require "active_support/current_attributes"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
##
|
5
|
+
# Automatically save and load any current attributes in the execution context
|
6
|
+
# so context attributes "flow" from Rails actions into any associated jobs.
|
7
|
+
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
8
|
+
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
9
|
+
#
|
10
|
+
# For multiple current attributes, pass an array of current attributes.
|
11
|
+
#
|
12
|
+
# @example
|
13
|
+
#
|
14
|
+
# # in your initializer
|
15
|
+
# require "sidekiq/middleware/current_attributes"
|
16
|
+
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
17
|
+
# # or multiple current attributes
|
18
|
+
# Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
|
19
|
+
#
|
20
|
+
module CurrentAttributes
|
21
|
+
class Save
|
22
|
+
include Sidekiq::ClientMiddleware
|
23
|
+
|
24
|
+
def initialize(cattrs)
|
25
|
+
@cattrs = cattrs
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(_, job, _, _)
|
29
|
+
@cattrs.each do |(key, strklass)|
|
30
|
+
if !job.has_key?(key)
|
31
|
+
attrs = strklass.constantize.attributes
|
32
|
+
# Retries can push the job N times, we don't
|
33
|
+
# want retries to reset cattr. #5692, #5090
|
34
|
+
job[key] = attrs if attrs.any?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
yield
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Load
|
42
|
+
include Sidekiq::ServerMiddleware
|
43
|
+
|
44
|
+
def initialize(cattrs)
|
45
|
+
@cattrs = cattrs
|
46
|
+
end
|
47
|
+
|
48
|
+
def call(_, job, _, &block)
|
49
|
+
cattrs_to_reset = []
|
50
|
+
|
51
|
+
@cattrs.each do |(key, strklass)|
|
52
|
+
if job.has_key?(key)
|
53
|
+
constklass = strklass.constantize
|
54
|
+
cattrs_to_reset << constklass
|
55
|
+
|
56
|
+
job[key].each do |(attribute, value)|
|
57
|
+
constklass.public_send("#{attribute}=", value)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
yield
|
63
|
+
ensure
|
64
|
+
cattrs_to_reset.each(&:reset)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class << self
|
69
|
+
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
70
|
+
cattrs = build_cattrs_hash(klass_or_array)
|
71
|
+
|
72
|
+
config.client_middleware.add Save, cattrs
|
73
|
+
config.server_middleware.add Load, cattrs
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def build_cattrs_hash(klass_or_array)
|
79
|
+
if klass_or_array.is_a?(Array)
|
80
|
+
{}.tap do |hash|
|
81
|
+
klass_or_array.each_with_index do |klass, index|
|
82
|
+
hash[key_at(index)] = klass.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
else
|
86
|
+
{key_at(0) => klass_or_array.to_s}
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def key_at(index)
|
91
|
+
(index == 0) ? "cattr" : "cattr_#{index}"
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Simple middleware to save the current locale and restore it when the job executes.
|
3
5
|
# Use it by requiring it in your initializer:
|
@@ -8,19 +10,18 @@ module Sidekiq::Middleware::I18n
|
|
8
10
|
# Get the current locale and store it in the message
|
9
11
|
# to be sent to Sidekiq.
|
10
12
|
class Client
|
11
|
-
|
12
|
-
|
13
|
+
include Sidekiq::ClientMiddleware
|
14
|
+
def call(_jobclass, job, _queue, _redis)
|
15
|
+
job["locale"] ||= I18n.locale
|
13
16
|
yield
|
14
17
|
end
|
15
18
|
end
|
16
19
|
|
17
20
|
# Pull the msg locale out and set the current thread to use it.
|
18
21
|
class Server
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
ensure
|
23
|
-
I18n.locale = I18n.default_locale
|
22
|
+
include Sidekiq::ServerMiddleware
|
23
|
+
def call(_jobclass, job, _queue, &block)
|
24
|
+
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
24
25
|
end
|
25
26
|
end
|
26
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
# Server-side middleware must import this Module in order
|
3
|
+
# to get access to server resources during `call`.
|
4
|
+
module ServerMiddleware
|
5
|
+
attr_accessor :config
|
6
|
+
def redis_pool
|
7
|
+
config.redis_pool
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
config.logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def redis(&block)
|
15
|
+
config.redis(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# no difference for now
|
20
|
+
ClientMiddleware = ServerMiddleware
|
21
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "fileutils"
|
4
|
+
require "sidekiq/api"
|
5
|
+
|
6
|
+
class Sidekiq::Monitor
|
7
|
+
class Status
|
8
|
+
VALID_SECTIONS = %w[all version overview processes queues]
|
9
|
+
COL_PAD = 2
|
10
|
+
|
11
|
+
def display(section = nil)
|
12
|
+
section ||= "all"
|
13
|
+
unless VALID_SECTIONS.include? section
|
14
|
+
puts "I don't know how to check the status of '#{section}'!"
|
15
|
+
puts "Try one of these: #{VALID_SECTIONS.join(", ")}"
|
16
|
+
return
|
17
|
+
end
|
18
|
+
send(section)
|
19
|
+
end
|
20
|
+
|
21
|
+
def all
|
22
|
+
version
|
23
|
+
puts
|
24
|
+
overview
|
25
|
+
puts
|
26
|
+
processes
|
27
|
+
puts
|
28
|
+
queues
|
29
|
+
end
|
30
|
+
|
31
|
+
def version
|
32
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
33
|
+
puts Time.now.utc
|
34
|
+
end
|
35
|
+
|
36
|
+
def overview
|
37
|
+
puts "---- Overview ----"
|
38
|
+
puts " Processed: #{delimit stats.processed}"
|
39
|
+
puts " Failed: #{delimit stats.failed}"
|
40
|
+
puts " Busy: #{delimit stats.workers_size}"
|
41
|
+
puts " Enqueued: #{delimit stats.enqueued}"
|
42
|
+
puts " Retries: #{delimit stats.retry_size}"
|
43
|
+
puts " Scheduled: #{delimit stats.scheduled_size}"
|
44
|
+
puts " Dead: #{delimit stats.dead_size}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def processes
|
48
|
+
puts "---- Processes (#{process_set.size}) ----"
|
49
|
+
process_set.each_with_index do |process, index|
|
50
|
+
# Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
|
51
|
+
#
|
52
|
+
# Before:
|
53
|
+
# ["default", "critical"]
|
54
|
+
#
|
55
|
+
# After:
|
56
|
+
# {"default" => 1, "critical" => 10}
|
57
|
+
queues =
|
58
|
+
if process["weights"]
|
59
|
+
process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
|
60
|
+
else
|
61
|
+
process["queues"].sort
|
62
|
+
end
|
63
|
+
|
64
|
+
puts "#{process["identity"]} #{tags_for(process)}"
|
65
|
+
puts " Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
|
66
|
+
puts " Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
|
67
|
+
puts " Queues: #{split_multiline(queues, pad: 11)}"
|
68
|
+
puts " Version: #{process["version"] || "Unknown"}" if process["version"] != Sidekiq::VERSION
|
69
|
+
puts "" unless (index + 1) == process_set.size
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def queues
|
74
|
+
puts "---- Queues (#{queue_data.size}) ----"
|
75
|
+
columns = {
|
76
|
+
name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
77
|
+
size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
78
|
+
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
79
|
+
}
|
80
|
+
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
81
|
+
puts
|
82
|
+
queue_data.each do |q|
|
83
|
+
columns.each do |col, (dir, width)|
|
84
|
+
print q.send(col).public_send(dir, width)
|
85
|
+
end
|
86
|
+
puts
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def delimit(number)
|
93
|
+
number.to_s.reverse.scan(/.{1,3}/).join(",").reverse
|
94
|
+
end
|
95
|
+
|
96
|
+
def split_multiline(values, opts = {})
|
97
|
+
return "none" unless values
|
98
|
+
pad = opts[:pad] || 0
|
99
|
+
max_length = opts[:max_length] || (80 - pad)
|
100
|
+
out = []
|
101
|
+
line = ""
|
102
|
+
values.each do |value|
|
103
|
+
if (line.length + value.length) > max_length
|
104
|
+
out << line
|
105
|
+
line = " " * pad
|
106
|
+
end
|
107
|
+
line << value + ", "
|
108
|
+
end
|
109
|
+
out << line[0..-3]
|
110
|
+
out.join("\n")
|
111
|
+
end
|
112
|
+
|
113
|
+
def tags_for(process)
|
114
|
+
tags = [
|
115
|
+
process["tag"],
|
116
|
+
process["labels"],
|
117
|
+
((process["quiet"] == "true") ? "quiet" : nil)
|
118
|
+
].flatten.compact
|
119
|
+
tags.any? ? "[#{tags.join("] [")}]" : nil
|
120
|
+
end
|
121
|
+
|
122
|
+
def time_ago(timestamp)
|
123
|
+
seconds = Time.now - Time.at(timestamp)
|
124
|
+
return "just now" if seconds < 60
|
125
|
+
return "a minute ago" if seconds < 120
|
126
|
+
return "#{seconds.floor / 60} minutes ago" if seconds < 3600
|
127
|
+
return "an hour ago" if seconds < 7200
|
128
|
+
"#{seconds.floor / 60 / 60} hours ago"
|
129
|
+
end
|
130
|
+
|
131
|
+
QUEUE_STRUCT = Struct.new(:name, :size, :latency)
|
132
|
+
def queue_data
|
133
|
+
@queue_data ||= Sidekiq::Queue.all.map { |q|
|
134
|
+
QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf("%#.2f", q.latency))
|
135
|
+
}
|
136
|
+
end
|
137
|
+
|
138
|
+
def process_set
|
139
|
+
@process_set ||= Sidekiq::ProcessSet.new
|
140
|
+
end
|
141
|
+
|
142
|
+
def stats
|
143
|
+
@stats ||= Sidekiq::Stats.new
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|