sidekiq 6.4.0 → 7.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +232 -12
- data/README.md +44 -31
- data/bin/sidekiq +4 -9
- data/bin/sidekiqload +207 -117
- data/bin/sidekiqmon +4 -1
- data/lib/sidekiq/api.rb +329 -188
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +85 -81
- data/lib/sidekiq/client.rb +98 -58
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +278 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +23 -24
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +80 -56
- data/lib/sidekiq/job_util.rb +60 -20
- data/lib/sidekiq/launcher.rb +103 -95
- data/lib/sidekiq/logger.rb +9 -44
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +58 -20
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +22 -10
- data/lib/sidekiq/redis_client_adapter.rb +96 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +66 -38
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +40 -9
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +35 -21
- data/lib/sidekiq/web.rb +10 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -206
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +66 -297
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +44 -28
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +56 -27
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -364
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -15,11 +15,11 @@ module Sidekiq
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def halt(res)
|
18
|
-
throw :halt, [res, {"
|
18
|
+
throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
22
|
-
throw :halt, [302, {"
|
22
|
+
throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
25
|
def params
|
@@ -68,7 +68,7 @@ module Sidekiq
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def json(payload)
|
71
|
-
[200, {"
|
71
|
+
[200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
72
72
|
end
|
73
73
|
|
74
74
|
def initialize(env, block)
|
@@ -20,6 +20,12 @@ module Sidekiq
|
|
20
20
|
"worker-src 'self'",
|
21
21
|
"base-uri 'self'"
|
22
22
|
].join("; ").freeze
|
23
|
+
METRICS_PERIODS = {
|
24
|
+
"1h" => 60,
|
25
|
+
"2h" => 120,
|
26
|
+
"4h" => 240,
|
27
|
+
"8h" => 480
|
28
|
+
}
|
23
29
|
|
24
30
|
def initialize(klass)
|
25
31
|
@klass = klass
|
@@ -60,17 +66,42 @@ module Sidekiq
|
|
60
66
|
erb(:dashboard)
|
61
67
|
end
|
62
68
|
|
69
|
+
get "/metrics" do
|
70
|
+
q = Sidekiq::Metrics::Query.new
|
71
|
+
@period = h((params[:period] || "")[0..1])
|
72
|
+
@periods = METRICS_PERIODS
|
73
|
+
minutes = @periods.fetch(@period, @periods.values.first)
|
74
|
+
@query_result = q.top_jobs(minutes: minutes)
|
75
|
+
erb(:metrics)
|
76
|
+
end
|
77
|
+
|
78
|
+
get "/metrics/:name" do
|
79
|
+
@name = route_params[:name]
|
80
|
+
@period = h((params[:period] || "")[0..1])
|
81
|
+
q = Sidekiq::Metrics::Query.new
|
82
|
+
@periods = METRICS_PERIODS
|
83
|
+
minutes = @periods.fetch(@period, @periods.values.first)
|
84
|
+
@query_result = q.for_job(@name, minutes: minutes)
|
85
|
+
erb(:metrics_for_job)
|
86
|
+
end
|
87
|
+
|
63
88
|
get "/busy" do
|
89
|
+
@count = (params["count"] || 100).to_i
|
90
|
+
(@current_page, @total_size, @workset) = page_items(workset, params["page"], @count)
|
91
|
+
|
64
92
|
erb(:busy)
|
65
93
|
end
|
66
94
|
|
67
95
|
post "/busy" do
|
68
96
|
if params["identity"]
|
69
|
-
|
70
|
-
|
71
|
-
|
97
|
+
pro = Sidekiq::ProcessSet[params["identity"]]
|
98
|
+
|
99
|
+
pro.quiet! if params["quiet"]
|
100
|
+
pro.stop! if params["stop"]
|
72
101
|
else
|
73
102
|
processes.each do |pro|
|
103
|
+
next if pro.embedded?
|
104
|
+
|
74
105
|
pro.quiet! if params["quiet"]
|
75
106
|
pro.stop! if params["stop"]
|
76
107
|
end
|
@@ -294,12 +325,12 @@ module Sidekiq
|
|
294
325
|
end
|
295
326
|
|
296
327
|
get "/stats/queues" do
|
297
|
-
json Sidekiq::Stats
|
328
|
+
json Sidekiq::Stats.new.queues
|
298
329
|
end
|
299
330
|
|
300
331
|
def call(env)
|
301
332
|
action = self.class.match(env)
|
302
|
-
return [404, {"
|
333
|
+
return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
|
303
334
|
|
304
335
|
app = @klass
|
305
336
|
resp = catch(:halt) do
|
@@ -316,10 +347,10 @@ module Sidekiq
|
|
316
347
|
else
|
317
348
|
# rendered content goes here
|
318
349
|
headers = {
|
319
|
-
"
|
320
|
-
"
|
321
|
-
"
|
322
|
-
"
|
350
|
+
"content-type" => "text/html",
|
351
|
+
"cache-control" => "private, no-store",
|
352
|
+
"content-language" => action.locale,
|
353
|
+
"content-security-policy" => CSP_HEADER
|
323
354
|
}
|
324
355
|
# we'll let Rack calculate Content-Length for us.
|
325
356
|
[200, headers, [resp]]
|
@@ -143,7 +143,7 @@ module Sidekiq
|
|
143
143
|
one_time_pad = SecureRandom.random_bytes(token.length)
|
144
144
|
encrypted_token = xor_byte_strings(one_time_pad, token)
|
145
145
|
masked_token = one_time_pad + encrypted_token
|
146
|
-
Base64.
|
146
|
+
Base64.urlsafe_encode64(masked_token)
|
147
147
|
end
|
148
148
|
|
149
149
|
# Essentially the inverse of +mask_token+.
|
@@ -152,7 +152,7 @@ module Sidekiq
|
|
152
152
|
# value and decrypt it
|
153
153
|
token_length = masked_token.length / 2
|
154
154
|
one_time_pad = masked_token[0...token_length]
|
155
|
-
encrypted_token = masked_token[token_length
|
155
|
+
encrypted_token = masked_token[token_length..]
|
156
156
|
xor_byte_strings(one_time_pad, encrypted_token)
|
157
157
|
end
|
158
158
|
|
@@ -169,7 +169,7 @@ module Sidekiq
|
|
169
169
|
end
|
170
170
|
|
171
171
|
def decode_token(token)
|
172
|
-
Base64.
|
172
|
+
Base64.urlsafe_decode64(token)
|
173
173
|
end
|
174
174
|
|
175
175
|
def xor_byte_strings(s1, s2)
|
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -15,7 +15,7 @@ module Sidekiq
|
|
15
15
|
# so extensions can be localized
|
16
16
|
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
17
|
find_locale_files(lang).each do |file|
|
18
|
-
strs = YAML.
|
18
|
+
strs = YAML.safe_load(File.read(file))
|
19
19
|
global.merge!(strs[lang])
|
20
20
|
end
|
21
21
|
end
|
@@ -118,7 +118,7 @@ module Sidekiq
|
|
118
118
|
}.join(" ")
|
119
119
|
end
|
120
120
|
|
121
|
-
#
|
121
|
+
# sidekiq/sidekiq#3243
|
122
122
|
def unfiltered?
|
123
123
|
yield unless env["PATH_INFO"].start_with?("/filter/")
|
124
124
|
end
|
@@ -137,33 +137,50 @@ module Sidekiq
|
|
137
137
|
end
|
138
138
|
|
139
139
|
def sort_direction_label
|
140
|
-
params[:direction] == "asc" ? "↑" : "↓"
|
140
|
+
(params[:direction] == "asc") ? "↑" : "↓"
|
141
141
|
end
|
142
142
|
|
143
|
-
def
|
144
|
-
@
|
143
|
+
def workset
|
144
|
+
@work ||= Sidekiq::WorkSet.new
|
145
145
|
end
|
146
146
|
|
147
147
|
def processes
|
148
148
|
@processes ||= Sidekiq::ProcessSet.new
|
149
149
|
end
|
150
150
|
|
151
|
+
# Sorts processes by hostname following the natural sort order
|
152
|
+
def sorted_processes
|
153
|
+
@sorted_processes ||= begin
|
154
|
+
return processes unless processes.all? { |p| p["hostname"] }
|
155
|
+
|
156
|
+
processes.to_a.sort_by do |process|
|
157
|
+
# Kudos to `shurikk` on StackOverflow
|
158
|
+
# https://stackoverflow.com/a/15170063/575547
|
159
|
+
process["hostname"].split(/(\d+)/).map { |a| /\d+/.match?(a) ? a.to_i : a }
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def busy_weights(capsule_weights)
|
165
|
+
# backwards compat with 7.0.0, remove in 7.1
|
166
|
+
cw = [capsule_weights].flatten
|
167
|
+
cw.map { |hash|
|
168
|
+
hash.map { |name, weight| (weight > 0) ? +name << ": " << weight.to_s : name }.join(", ")
|
169
|
+
}.join("; ")
|
170
|
+
end
|
171
|
+
|
151
172
|
def stats
|
152
173
|
@stats ||= Sidekiq::Stats.new
|
153
174
|
end
|
154
175
|
|
155
|
-
def
|
176
|
+
def redis_url
|
156
177
|
Sidekiq.redis do |conn|
|
157
|
-
conn.
|
178
|
+
conn.config.server_url
|
158
179
|
end
|
159
180
|
end
|
160
181
|
|
161
|
-
def namespace
|
162
|
-
@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
|
163
|
-
end
|
164
|
-
|
165
182
|
def redis_info
|
166
|
-
Sidekiq.redis_info
|
183
|
+
Sidekiq.default_configuration.redis_info
|
167
184
|
end
|
168
185
|
|
169
186
|
def root_path
|
@@ -175,7 +192,7 @@ module Sidekiq
|
|
175
192
|
end
|
176
193
|
|
177
194
|
def current_status
|
178
|
-
|
195
|
+
(workset.size == 0) ? "idle" : "active"
|
179
196
|
end
|
180
197
|
|
181
198
|
def relative_time(time)
|
@@ -208,7 +225,7 @@ module Sidekiq
|
|
208
225
|
end
|
209
226
|
|
210
227
|
def truncate(text, truncate_after_chars = 2000)
|
211
|
-
truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
|
228
|
+
(truncate_after_chars && text.size > truncate_after_chars) ? "#{text[0..truncate_after_chars]}..." : text
|
212
229
|
end
|
213
230
|
|
214
231
|
def display_args(args, truncate_after_chars = 2000)
|
@@ -242,7 +259,7 @@ module Sidekiq
|
|
242
259
|
queue class args retry_count retried_at failed_at
|
243
260
|
jid error_message error_class backtrace
|
244
261
|
error_backtrace enqueued_at retry wrapped
|
245
|
-
created_at tags
|
262
|
+
created_at tags display_class
|
246
263
|
])
|
247
264
|
|
248
265
|
def retry_extra_items(retry_job)
|
@@ -301,7 +318,7 @@ module Sidekiq
|
|
301
318
|
end
|
302
319
|
|
303
320
|
def environment_title_prefix
|
304
|
-
environment = Sidekiq.
|
321
|
+
environment = Sidekiq.default_configuration[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
305
322
|
|
306
323
|
"[#{environment.upcase}] " unless environment == "production"
|
307
324
|
end
|
@@ -314,11 +331,8 @@ module Sidekiq
|
|
314
331
|
Time.now.utc.strftime("%H:%M:%S UTC")
|
315
332
|
end
|
316
333
|
|
317
|
-
def
|
318
|
-
|
319
|
-
namespace_suffix = namespace.nil? ? "" : "##{namespace}"
|
320
|
-
"#{redis_connection}#{namespace_suffix}"
|
321
|
-
end
|
334
|
+
def pollable?
|
335
|
+
!(current_path == "" || current_path.start_with?("metrics"))
|
322
336
|
end
|
323
337
|
|
324
338
|
def retry_or_delete_or_kill(job, params)
|
data/lib/sidekiq/web.rb
CHANGED
@@ -30,7 +30,8 @@ module Sidekiq
|
|
30
30
|
"Queues" => "queues",
|
31
31
|
"Retries" => "retries",
|
32
32
|
"Scheduled" => "scheduled",
|
33
|
-
"Dead" => "morgue"
|
33
|
+
"Dead" => "morgue",
|
34
|
+
"Metrics" => "metrics"
|
34
35
|
}
|
35
36
|
|
36
37
|
class << self
|
@@ -47,6 +48,10 @@ module Sidekiq
|
|
47
48
|
end
|
48
49
|
alias_method :tabs, :custom_tabs
|
49
50
|
|
51
|
+
def custom_job_info_rows
|
52
|
+
@custom_job_info_rows ||= []
|
53
|
+
end
|
54
|
+
|
50
55
|
def locales
|
51
56
|
@locales ||= LOCALES
|
52
57
|
end
|
@@ -75,14 +80,6 @@ module Sidekiq
|
|
75
80
|
send(:"#{attribute}=", value)
|
76
81
|
end
|
77
82
|
|
78
|
-
def sessions=(val)
|
79
|
-
puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
80
|
-
end
|
81
|
-
|
82
|
-
def session_secret=(val)
|
83
|
-
puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
84
|
-
end
|
85
|
-
|
86
83
|
attr_accessor :app_url, :redis_pool
|
87
84
|
attr_writer :locales, :views
|
88
85
|
end
|
@@ -129,10 +126,6 @@ module Sidekiq
|
|
129
126
|
send(:"#{attribute}=", value)
|
130
127
|
end
|
131
128
|
|
132
|
-
def sessions=(val)
|
133
|
-
puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
|
134
|
-
end
|
135
|
-
|
136
129
|
def self.register(extension)
|
137
130
|
extension.registered(WebApplication)
|
138
131
|
end
|
@@ -144,13 +137,13 @@ module Sidekiq
|
|
144
137
|
m = middlewares
|
145
138
|
|
146
139
|
rules = []
|
147
|
-
rules = [[:all, {"Cache-Control" => "
|
140
|
+
rules = [[:all, {"Cache-Control" => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
148
141
|
|
149
142
|
::Rack::Builder.new do
|
150
143
|
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
151
|
-
|
152
|
-
|
153
|
-
|
144
|
+
root: ASSETS,
|
145
|
+
cascade: true,
|
146
|
+
header_rules: rules
|
154
147
|
m.each { |middleware, block| use(*middleware, &block) }
|
155
148
|
use Sidekiq::Web::CsrfProtection unless $TESTING
|
156
149
|
run WebApplication.new(klass)
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
|
5
|
+
# Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
|
6
|
+
#
|
7
|
+
# The term "worker" is too generic and overly confusing, used in several
|
8
|
+
# different contexts meaning different things. Many people call a Sidekiq
|
9
|
+
# process a "worker". Some people call the thread that executes jobs a
|
10
|
+
# "worker". This change brings Sidekiq closer to ActiveJob where your job
|
11
|
+
# classes extend ApplicationJob.
|
12
|
+
Worker = Job
|
13
|
+
end
|