sidekiq 6.4.0 → 6.5.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +119 -1
- data/README.md +6 -1
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/sidekiq/api.rb +255 -100
- data/lib/sidekiq/cli.rb +60 -38
- data/lib/sidekiq/client.rb +44 -30
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +2 -2
- data/lib/sidekiq/extensions/action_mailer.rb +2 -2
- data/lib/sidekiq/extensions/active_record.rb +2 -2
- data/lib/sidekiq/extensions/class_methods.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
- data/lib/sidekiq/fetch.rb +20 -18
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +73 -52
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +58 -54
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +18 -12
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +19 -13
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -49
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +53 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +21 -5
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +21 -8
- data/lib/sidekiq/web.rb +8 -4
- data/lib/sidekiq/worker.rb +26 -20
- data/lib/sidekiq.rb +107 -31
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +59 -26
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application.css +45 -1
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/ja.yml +7 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +9 -4
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +34 -9
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq.rb
CHANGED
@@ -5,6 +5,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
5
5
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
|
+
require "sidekiq/transaction_aware_client"
|
8
9
|
require "sidekiq/worker"
|
9
10
|
require "sidekiq/job"
|
10
11
|
require "sidekiq/redis_connection"
|
@@ -33,18 +34,16 @@ module Sidekiq
|
|
33
34
|
startup: [],
|
34
35
|
quiet: [],
|
35
36
|
shutdown: [],
|
36
|
-
heartbeat
|
37
|
+
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
38
|
+
heartbeat: [],
|
39
|
+
# triggers on EVERY heartbeat call, every 10 seconds
|
40
|
+
beat: []
|
37
41
|
},
|
38
42
|
dead_max_jobs: 10_000,
|
39
43
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
40
44
|
reloader: proc { |&block| block.call }
|
41
45
|
}
|
42
46
|
|
43
|
-
DEFAULT_WORKER_OPTIONS = {
|
44
|
-
"retry" => true,
|
45
|
-
"queue" => "default"
|
46
|
-
}
|
47
|
-
|
48
47
|
FAKE_INFO = {
|
49
48
|
"redis_version" => "9.9.9",
|
50
49
|
"uptime_in_days" => "9999",
|
@@ -57,19 +56,84 @@ module Sidekiq
|
|
57
56
|
puts "Calm down, yo."
|
58
57
|
end
|
59
58
|
|
59
|
+
# config.concurrency = 5
|
60
|
+
def self.concurrency=(val)
|
61
|
+
self[:concurrency] = Integer(val)
|
62
|
+
end
|
63
|
+
|
64
|
+
# config.queues = %w( high default low ) # strict
|
65
|
+
# config.queues = %w( high,3 default,2 low,1 ) # weighted
|
66
|
+
# config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
|
67
|
+
#
|
68
|
+
# With weighted priority, queue will be checked first (weight / total) of the time.
|
69
|
+
# high will be checked first (3/6) or 50% of the time.
|
70
|
+
# I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
|
71
|
+
# are ridiculous and unnecessarily expensive. You can get random queue ordering
|
72
|
+
# by explicitly setting all weights to 1.
|
73
|
+
def self.queues=(val)
|
74
|
+
self[:queues] = Array(val).each_with_object([]) do |qstr, memo|
|
75
|
+
name, weight = qstr.split(",")
|
76
|
+
self[:strict] = false if weight.to_i > 0
|
77
|
+
[weight.to_i, 1].max.times do
|
78
|
+
memo << name
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
### Private APIs
|
84
|
+
def self.default_error_handler(ex, ctx)
|
85
|
+
logger.warn(dump_json(ctx)) unless ctx.empty?
|
86
|
+
logger.warn("#{ex.class.name}: #{ex.message}")
|
87
|
+
logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
88
|
+
end
|
89
|
+
|
90
|
+
# DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
|
91
|
+
# be referenced. It must be defined here, after the default_error_handler
|
92
|
+
# method is defined.
|
93
|
+
DEFAULT_ERROR_HANDLER = method(:default_error_handler)
|
94
|
+
|
95
|
+
@config = DEFAULTS.dup
|
60
96
|
def self.options
|
61
|
-
|
97
|
+
logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
|
98
|
+
@config
|
62
99
|
end
|
63
100
|
|
64
101
|
def self.options=(opts)
|
65
|
-
|
102
|
+
logger.warn "config.options = hash` is deprecated, use `config.merge!(hash)`: #{caller(1..2)}"
|
103
|
+
@config = opts
|
66
104
|
end
|
67
105
|
|
106
|
+
def self.[](key)
|
107
|
+
@config[key]
|
108
|
+
end
|
109
|
+
|
110
|
+
def self.[]=(key, val)
|
111
|
+
@config[key] = val
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.merge!(hash)
|
115
|
+
@config.merge!(hash)
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.fetch(*args, &block)
|
119
|
+
@config.fetch(*args, &block)
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.handle_exception(ex, ctx = {})
|
123
|
+
self[:error_handlers].each do |handler|
|
124
|
+
handler.call(ex, ctx)
|
125
|
+
rescue => ex
|
126
|
+
logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
127
|
+
logger.error ex
|
128
|
+
logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
###
|
132
|
+
|
68
133
|
##
|
69
134
|
# Configuration for Sidekiq server, use like:
|
70
135
|
#
|
71
136
|
# Sidekiq.configure_server do |config|
|
72
|
-
# config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
|
73
137
|
# config.server_middleware do |chain|
|
74
138
|
# chain.add MyServerHook
|
75
139
|
# end
|
@@ -82,7 +146,7 @@ module Sidekiq
|
|
82
146
|
# Configuration for Sidekiq client, use like:
|
83
147
|
#
|
84
148
|
# Sidekiq.configure_client do |config|
|
85
|
-
# config.redis = { :
|
149
|
+
# config.redis = { size: 1, url: 'redis://myhost:8877/0' }
|
86
150
|
# end
|
87
151
|
def self.configure_client
|
88
152
|
yield self unless server?
|
@@ -98,11 +162,12 @@ module Sidekiq
|
|
98
162
|
retryable = true
|
99
163
|
begin
|
100
164
|
yield conn
|
101
|
-
rescue
|
165
|
+
rescue RedisConnection.adapter::BaseError => ex
|
102
166
|
# 2550 Failover can cause the server to become a replica, need
|
103
167
|
# to disconnect and reopen the socket to get back to the primary.
|
104
168
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
105
169
|
# 4985 Use the same logic when a blocking command is force-unblocked
|
170
|
+
# The same retry logic is also used in client.rb
|
106
171
|
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
107
172
|
conn.disconnect!
|
108
173
|
retryable = false
|
@@ -122,7 +187,7 @@ module Sidekiq
|
|
122
187
|
else
|
123
188
|
conn.info
|
124
189
|
end
|
125
|
-
rescue
|
190
|
+
rescue RedisConnection.adapter::CommandError => ex
|
126
191
|
# 2850 return fake version when INFO command has (probably) been renamed
|
127
192
|
raise unless /unknown command/.match?(ex.message)
|
128
193
|
FAKE_INFO
|
@@ -130,19 +195,19 @@ module Sidekiq
|
|
130
195
|
end
|
131
196
|
|
132
197
|
def self.redis_pool
|
133
|
-
@redis ||=
|
198
|
+
@redis ||= RedisConnection.create
|
134
199
|
end
|
135
200
|
|
136
201
|
def self.redis=(hash)
|
137
202
|
@redis = if hash.is_a?(ConnectionPool)
|
138
203
|
hash
|
139
204
|
else
|
140
|
-
|
205
|
+
RedisConnection.create(hash)
|
141
206
|
end
|
142
207
|
end
|
143
208
|
|
144
209
|
def self.client_middleware
|
145
|
-
@client_chain ||= Middleware::Chain.new
|
210
|
+
@client_chain ||= Middleware::Chain.new(self)
|
146
211
|
yield @client_chain if block_given?
|
147
212
|
@client_chain
|
148
213
|
end
|
@@ -154,16 +219,23 @@ module Sidekiq
|
|
154
219
|
end
|
155
220
|
|
156
221
|
def self.default_server_middleware
|
157
|
-
Middleware::Chain.new
|
222
|
+
Middleware::Chain.new(self)
|
223
|
+
end
|
224
|
+
|
225
|
+
def self.default_worker_options=(hash) # deprecated
|
226
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
158
227
|
end
|
159
228
|
|
160
|
-
def self.
|
161
|
-
|
162
|
-
@default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
|
229
|
+
def self.default_job_options=(hash)
|
230
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
163
231
|
end
|
164
232
|
|
165
|
-
def self.default_worker_options
|
166
|
-
|
233
|
+
def self.default_worker_options # deprecated
|
234
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
235
|
+
end
|
236
|
+
|
237
|
+
def self.default_job_options
|
238
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
167
239
|
end
|
168
240
|
|
169
241
|
##
|
@@ -176,7 +248,7 @@ module Sidekiq
|
|
176
248
|
# end
|
177
249
|
# end
|
178
250
|
def self.death_handlers
|
179
|
-
|
251
|
+
self[:death_handlers]
|
180
252
|
end
|
181
253
|
|
182
254
|
def self.load_json(string)
|
@@ -201,7 +273,7 @@ module Sidekiq
|
|
201
273
|
end
|
202
274
|
|
203
275
|
def self.logger
|
204
|
-
@logger ||= Sidekiq::Logger.new($stdout, level:
|
276
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: :info)
|
205
277
|
end
|
206
278
|
|
207
279
|
def self.logger=(logger)
|
@@ -219,13 +291,17 @@ module Sidekiq
|
|
219
291
|
defined?(Sidekiq::Pro)
|
220
292
|
end
|
221
293
|
|
294
|
+
def self.ent?
|
295
|
+
defined?(Sidekiq::Enterprise)
|
296
|
+
end
|
297
|
+
|
222
298
|
# How frequently Redis should be checked by a random Sidekiq process for
|
223
299
|
# scheduled and retriable jobs. Each individual process will take turns by
|
224
300
|
# waiting some multiple of this value.
|
225
301
|
#
|
226
302
|
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
227
303
|
def self.average_scheduled_poll_interval=(interval)
|
228
|
-
|
304
|
+
self[:average_scheduled_poll_interval] = interval
|
229
305
|
end
|
230
306
|
|
231
307
|
# Register a proc to handle any error which occurs within the Sidekiq process.
|
@@ -236,7 +312,7 @@ module Sidekiq
|
|
236
312
|
#
|
237
313
|
# The default error handler logs errors to Sidekiq.logger.
|
238
314
|
def self.error_handlers
|
239
|
-
|
315
|
+
self[:error_handlers]
|
240
316
|
end
|
241
317
|
|
242
318
|
# Register a block to run at a point in the Sidekiq lifecycle.
|
@@ -249,20 +325,20 @@ module Sidekiq
|
|
249
325
|
# end
|
250
326
|
def self.on(event, &block)
|
251
327
|
raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
|
252
|
-
raise ArgumentError, "Invalid event name: #{event}" unless
|
253
|
-
|
328
|
+
raise ArgumentError, "Invalid event name: #{event}" unless self[:lifecycle_events].key?(event)
|
329
|
+
self[:lifecycle_events][event] << block
|
254
330
|
end
|
255
331
|
|
256
332
|
def self.strict_args!(mode = :raise)
|
257
|
-
|
333
|
+
self[:on_complex_arguments] = mode
|
258
334
|
end
|
259
335
|
|
260
|
-
# We are shutting down Sidekiq but what about
|
336
|
+
# We are shutting down Sidekiq but what about threads that
|
261
337
|
# are working on some long job? This error is
|
262
|
-
# raised in
|
338
|
+
# raised in jobs that have not finished within the hard
|
263
339
|
# timeout limit. This is needed to rollback db transactions,
|
264
340
|
# otherwise Ruby's Thread#kill will commit. See #377.
|
265
|
-
# DO NOT RESCUE THIS ERROR IN YOUR
|
341
|
+
# DO NOT RESCUE THIS ERROR IN YOUR JOBS
|
266
342
|
class Shutdown < Interrupt; end
|
267
343
|
end
|
268
344
|
|
data/sidekiq.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
22
22
|
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
23
|
}
|
24
24
|
|
25
|
-
gem.add_dependency "redis", ">= 4.
|
26
|
-
gem.add_dependency "connection_pool", ">= 2.2.
|
25
|
+
gem.add_dependency "redis", ["<5", ">= 4.5.0"]
|
26
|
+
gem.add_dependency "connection_pool", ["<3", ">= 2.2.5"]
|
27
27
|
gem.add_dependency "rack", "~> 2.0"
|
28
28
|
end
|
@@ -9,7 +9,9 @@ var ready = (callback) => {
|
|
9
9
|
else document.addEventListener("DOMContentLoaded", callback);
|
10
10
|
}
|
11
11
|
|
12
|
-
ready(
|
12
|
+
ready(addListeners)
|
13
|
+
|
14
|
+
function addListeners() {
|
13
15
|
document.querySelectorAll(".check_all").forEach(node => {
|
14
16
|
node.addEventListener("click", event => {
|
15
17
|
node.closest('table').querySelectorAll('input[type=checkbox]').forEach(inp => { inp.checked = !!node.checked; });
|
@@ -26,42 +28,48 @@ ready(() => {
|
|
26
28
|
})
|
27
29
|
|
28
30
|
document.querySelectorAll("[data-toggle]").forEach(node => {
|
29
|
-
node.addEventListener("click",
|
30
|
-
var targName = node.getAttribute("data-toggle");
|
31
|
-
var full = document.getElementById(targName + "_full");
|
32
|
-
if (full.style.display == "block") {
|
33
|
-
full.style.display = 'none';
|
34
|
-
} else {
|
35
|
-
full.style.display = 'block';
|
36
|
-
}
|
37
|
-
})
|
31
|
+
node.addEventListener("click", addDataToggleListeners)
|
38
32
|
})
|
39
33
|
|
40
34
|
updateFuzzyTimes();
|
35
|
+
setLivePollFromUrl();
|
41
36
|
|
42
37
|
var buttons = document.querySelectorAll(".live-poll");
|
43
38
|
if (buttons.length > 0) {
|
44
39
|
buttons.forEach(node => {
|
45
|
-
node.addEventListener("click",
|
46
|
-
if (localStorage.sidekiqLivePoll == "enabled") {
|
47
|
-
localStorage.sidekiqLivePoll = "disabled";
|
48
|
-
clearTimeout(livePollTimer);
|
49
|
-
livePollTimer = null;
|
50
|
-
} else {
|
51
|
-
localStorage.sidekiqLivePoll = "enabled";
|
52
|
-
livePollCallback();
|
53
|
-
}
|
54
|
-
|
55
|
-
updateLivePollButton();
|
56
|
-
})
|
40
|
+
node.addEventListener("click", addPollingListeners)
|
57
41
|
});
|
58
42
|
|
59
43
|
updateLivePollButton();
|
60
|
-
if (localStorage.sidekiqLivePoll == "enabled") {
|
44
|
+
if (localStorage.sidekiqLivePoll == "enabled" && !livePollTimer) {
|
61
45
|
scheduleLivePoll();
|
62
46
|
}
|
63
47
|
}
|
64
|
-
}
|
48
|
+
}
|
49
|
+
|
50
|
+
function addPollingListeners(_event) {
|
51
|
+
if (localStorage.sidekiqLivePoll == "enabled") {
|
52
|
+
localStorage.sidekiqLivePoll = "disabled";
|
53
|
+
clearTimeout(livePollTimer);
|
54
|
+
livePollTimer = null;
|
55
|
+
} else {
|
56
|
+
localStorage.sidekiqLivePoll = "enabled";
|
57
|
+
livePollCallback();
|
58
|
+
}
|
59
|
+
|
60
|
+
updateLivePollButton();
|
61
|
+
}
|
62
|
+
|
63
|
+
function addDataToggleListeners(event) {
|
64
|
+
var source = event.target || event.srcElement;
|
65
|
+
var targName = source.getAttribute("data-toggle");
|
66
|
+
var full = document.getElementById(targName);
|
67
|
+
if (full.style.display == "block") {
|
68
|
+
full.style.display = 'none';
|
69
|
+
} else {
|
70
|
+
full.style.display = 'block';
|
71
|
+
}
|
72
|
+
}
|
65
73
|
|
66
74
|
function updateFuzzyTimes() {
|
67
75
|
var locale = document.body.getAttribute("data-locale");
|
@@ -76,6 +84,14 @@ function updateFuzzyTimes() {
|
|
76
84
|
t.cancel();
|
77
85
|
}
|
78
86
|
|
87
|
+
function setLivePollFromUrl() {
|
88
|
+
var url_params = new URL(window.location.href).searchParams
|
89
|
+
|
90
|
+
if (url_params.get("poll") == "true") {
|
91
|
+
localStorage.sidekiqLivePoll = "enabled";
|
92
|
+
}
|
93
|
+
}
|
94
|
+
|
79
95
|
function updateLivePollButton() {
|
80
96
|
if (localStorage.sidekiqLivePoll == "enabled") {
|
81
97
|
document.querySelectorAll('.live-poll-stop').forEach(box => { box.style.display = "inline-block" })
|
@@ -89,11 +105,24 @@ function updateLivePollButton() {
|
|
89
105
|
function livePollCallback() {
|
90
106
|
clearTimeout(livePollTimer);
|
91
107
|
|
92
|
-
fetch(window.location.href)
|
108
|
+
fetch(window.location.href)
|
109
|
+
.then(checkResponse)
|
110
|
+
.then(resp => resp.text())
|
111
|
+
.then(replacePage)
|
112
|
+
.catch(showError)
|
113
|
+
.finally(scheduleLivePoll)
|
114
|
+
}
|
115
|
+
|
116
|
+
function checkResponse(resp) {
|
117
|
+
if (!resp.ok) {
|
118
|
+
throw response.error();
|
119
|
+
}
|
120
|
+
return resp
|
93
121
|
}
|
94
122
|
|
95
123
|
function scheduleLivePoll() {
|
96
124
|
let ti = parseInt(localStorage.sidekiqTimeInterval) || 5000;
|
125
|
+
if (ti < 2000) { ti = 2000 }
|
97
126
|
livePollTimer = setTimeout(livePollCallback, ti);
|
98
127
|
}
|
99
128
|
|
@@ -107,5 +136,9 @@ function replacePage(text) {
|
|
107
136
|
var header_status = doc.querySelector('.status')
|
108
137
|
document.querySelector('.status').replaceWith(header_status)
|
109
138
|
|
110
|
-
|
139
|
+
addListeners();
|
140
|
+
}
|
141
|
+
|
142
|
+
function showError(error) {
|
143
|
+
console.error(error)
|
111
144
|
}
|