sidekiq 6.4.1 → 6.5.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +81 -1
- data/README.md +1 -1
- data/bin/sidekiqload +16 -10
- data/lib/sidekiq/api.rb +189 -58
- data/lib/sidekiq/cli.rb +39 -37
- data/lib/sidekiq/client.rb +26 -27
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +1 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
- data/lib/sidekiq/fetch.rb +18 -16
- data/lib/sidekiq/job_retry.rb +73 -52
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +37 -33
- data/lib/sidekiq/logger.rb +5 -19
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +10 -4
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +2 -2
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -49
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +12 -17
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +18 -5
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +28 -5
- data/lib/sidekiq/web.rb +5 -1
- data/lib/sidekiq/worker.rb +18 -13
- data/lib/sidekiq.rb +106 -31
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application.css +45 -3
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +4 -4
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +27 -8
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq.rb
CHANGED
@@ -5,6 +5,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
5
5
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
|
+
require "sidekiq/transaction_aware_client"
|
8
9
|
require "sidekiq/worker"
|
9
10
|
require "sidekiq/job"
|
10
11
|
require "sidekiq/redis_connection"
|
@@ -33,18 +34,16 @@ module Sidekiq
|
|
33
34
|
startup: [],
|
34
35
|
quiet: [],
|
35
36
|
shutdown: [],
|
36
|
-
heartbeat
|
37
|
+
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
38
|
+
heartbeat: [],
|
39
|
+
# triggers on EVERY heartbeat call, every 10 seconds
|
40
|
+
beat: []
|
37
41
|
},
|
38
42
|
dead_max_jobs: 10_000,
|
39
43
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
40
44
|
reloader: proc { |&block| block.call }
|
41
45
|
}
|
42
46
|
|
43
|
-
DEFAULT_WORKER_OPTIONS = {
|
44
|
-
"retry" => true,
|
45
|
-
"queue" => "default"
|
46
|
-
}
|
47
|
-
|
48
47
|
FAKE_INFO = {
|
49
48
|
"redis_version" => "9.9.9",
|
50
49
|
"uptime_in_days" => "9999",
|
@@ -57,19 +56,84 @@ module Sidekiq
|
|
57
56
|
puts "Calm down, yo."
|
58
57
|
end
|
59
58
|
|
59
|
+
# config.concurrency = 5
|
60
|
+
def self.concurrency=(val)
|
61
|
+
self[:concurrency] = Integer(val)
|
62
|
+
end
|
63
|
+
|
64
|
+
# config.queues = %w( high default low ) # strict
|
65
|
+
# config.queues = %w( high,3 default,2 low,1 ) # weighted
|
66
|
+
# config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
|
67
|
+
#
|
68
|
+
# With weighted priority, queue will be checked first (weight / total) of the time.
|
69
|
+
# high will be checked first (3/6) or 50% of the time.
|
70
|
+
# I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
|
71
|
+
# are ridiculous and unnecessarily expensive. You can get random queue ordering
|
72
|
+
# by explicitly setting all weights to 1.
|
73
|
+
def self.queues=(val)
|
74
|
+
self[:queues] = Array(val).each_with_object([]) do |qstr, memo|
|
75
|
+
name, weight = qstr.split(",")
|
76
|
+
self[:strict] = false if weight.to_i > 0
|
77
|
+
[weight.to_i, 1].max.times do
|
78
|
+
memo << name
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
### Private APIs
|
84
|
+
def self.default_error_handler(ex, ctx)
|
85
|
+
logger.warn(dump_json(ctx)) unless ctx.empty?
|
86
|
+
logger.warn("#{ex.class.name}: #{ex.message}")
|
87
|
+
logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
88
|
+
end
|
89
|
+
|
90
|
+
# DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
|
91
|
+
# be referenced. It must be defined here, after the default_error_handler
|
92
|
+
# method is defined.
|
93
|
+
DEFAULT_ERROR_HANDLER = method(:default_error_handler)
|
94
|
+
|
95
|
+
@config = DEFAULTS.dup
|
60
96
|
def self.options
|
61
|
-
|
97
|
+
logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
|
98
|
+
@config
|
62
99
|
end
|
63
100
|
|
64
101
|
def self.options=(opts)
|
65
|
-
|
102
|
+
logger.warn "config.options = hash` is deprecated, use `config.merge!(hash)`: #{caller(1..2)}"
|
103
|
+
@config = opts
|
66
104
|
end
|
67
105
|
|
106
|
+
def self.[](key)
|
107
|
+
@config[key]
|
108
|
+
end
|
109
|
+
|
110
|
+
def self.[]=(key, val)
|
111
|
+
@config[key] = val
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.merge!(hash)
|
115
|
+
@config.merge!(hash)
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.fetch(*args, &block)
|
119
|
+
@config.fetch(*args, &block)
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.handle_exception(ex, ctx = {})
|
123
|
+
self[:error_handlers].each do |handler|
|
124
|
+
handler.call(ex, ctx)
|
125
|
+
rescue => ex
|
126
|
+
logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
127
|
+
logger.error ex
|
128
|
+
logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
###
|
132
|
+
|
68
133
|
##
|
69
134
|
# Configuration for Sidekiq server, use like:
|
70
135
|
#
|
71
136
|
# Sidekiq.configure_server do |config|
|
72
|
-
# config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
|
73
137
|
# config.server_middleware do |chain|
|
74
138
|
# chain.add MyServerHook
|
75
139
|
# end
|
@@ -82,7 +146,7 @@ module Sidekiq
|
|
82
146
|
# Configuration for Sidekiq client, use like:
|
83
147
|
#
|
84
148
|
# Sidekiq.configure_client do |config|
|
85
|
-
# config.redis = { :
|
149
|
+
# config.redis = { size: 1, url: 'redis://myhost:8877/0' }
|
86
150
|
# end
|
87
151
|
def self.configure_client
|
88
152
|
yield self unless server?
|
@@ -98,7 +162,7 @@ module Sidekiq
|
|
98
162
|
retryable = true
|
99
163
|
begin
|
100
164
|
yield conn
|
101
|
-
rescue
|
165
|
+
rescue RedisConnection.adapter::BaseError => ex
|
102
166
|
# 2550 Failover can cause the server to become a replica, need
|
103
167
|
# to disconnect and reopen the socket to get back to the primary.
|
104
168
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
@@ -123,7 +187,7 @@ module Sidekiq
|
|
123
187
|
else
|
124
188
|
conn.info
|
125
189
|
end
|
126
|
-
rescue
|
190
|
+
rescue RedisConnection.adapter::CommandError => ex
|
127
191
|
# 2850 return fake version when INFO command has (probably) been renamed
|
128
192
|
raise unless /unknown command/.match?(ex.message)
|
129
193
|
FAKE_INFO
|
@@ -131,19 +195,19 @@ module Sidekiq
|
|
131
195
|
end
|
132
196
|
|
133
197
|
def self.redis_pool
|
134
|
-
@redis ||=
|
198
|
+
@redis ||= RedisConnection.create
|
135
199
|
end
|
136
200
|
|
137
201
|
def self.redis=(hash)
|
138
202
|
@redis = if hash.is_a?(ConnectionPool)
|
139
203
|
hash
|
140
204
|
else
|
141
|
-
|
205
|
+
RedisConnection.create(hash)
|
142
206
|
end
|
143
207
|
end
|
144
208
|
|
145
209
|
def self.client_middleware
|
146
|
-
@client_chain ||= Middleware::Chain.new
|
210
|
+
@client_chain ||= Middleware::Chain.new(self)
|
147
211
|
yield @client_chain if block_given?
|
148
212
|
@client_chain
|
149
213
|
end
|
@@ -155,16 +219,23 @@ module Sidekiq
|
|
155
219
|
end
|
156
220
|
|
157
221
|
def self.default_server_middleware
|
158
|
-
Middleware::Chain.new
|
222
|
+
Middleware::Chain.new(self)
|
223
|
+
end
|
224
|
+
|
225
|
+
def self.default_worker_options=(hash) # deprecated
|
226
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
159
227
|
end
|
160
228
|
|
161
|
-
def self.
|
162
|
-
|
163
|
-
@default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
|
229
|
+
def self.default_job_options=(hash)
|
230
|
+
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
164
231
|
end
|
165
232
|
|
166
|
-
def self.default_worker_options
|
167
|
-
|
233
|
+
def self.default_worker_options # deprecated
|
234
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
235
|
+
end
|
236
|
+
|
237
|
+
def self.default_job_options
|
238
|
+
@default_job_options ||= {"retry" => true, "queue" => "default"}
|
168
239
|
end
|
169
240
|
|
170
241
|
##
|
@@ -177,7 +248,7 @@ module Sidekiq
|
|
177
248
|
# end
|
178
249
|
# end
|
179
250
|
def self.death_handlers
|
180
|
-
|
251
|
+
self[:death_handlers]
|
181
252
|
end
|
182
253
|
|
183
254
|
def self.load_json(string)
|
@@ -202,7 +273,7 @@ module Sidekiq
|
|
202
273
|
end
|
203
274
|
|
204
275
|
def self.logger
|
205
|
-
@logger ||= Sidekiq::Logger.new($stdout, level:
|
276
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: :info)
|
206
277
|
end
|
207
278
|
|
208
279
|
def self.logger=(logger)
|
@@ -220,13 +291,17 @@ module Sidekiq
|
|
220
291
|
defined?(Sidekiq::Pro)
|
221
292
|
end
|
222
293
|
|
294
|
+
def self.ent?
|
295
|
+
defined?(Sidekiq::Enterprise)
|
296
|
+
end
|
297
|
+
|
223
298
|
# How frequently Redis should be checked by a random Sidekiq process for
|
224
299
|
# scheduled and retriable jobs. Each individual process will take turns by
|
225
300
|
# waiting some multiple of this value.
|
226
301
|
#
|
227
302
|
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
228
303
|
def self.average_scheduled_poll_interval=(interval)
|
229
|
-
|
304
|
+
self[:average_scheduled_poll_interval] = interval
|
230
305
|
end
|
231
306
|
|
232
307
|
# Register a proc to handle any error which occurs within the Sidekiq process.
|
@@ -237,7 +312,7 @@ module Sidekiq
|
|
237
312
|
#
|
238
313
|
# The default error handler logs errors to Sidekiq.logger.
|
239
314
|
def self.error_handlers
|
240
|
-
|
315
|
+
self[:error_handlers]
|
241
316
|
end
|
242
317
|
|
243
318
|
# Register a block to run at a point in the Sidekiq lifecycle.
|
@@ -250,20 +325,20 @@ module Sidekiq
|
|
250
325
|
# end
|
251
326
|
def self.on(event, &block)
|
252
327
|
raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
|
253
|
-
raise ArgumentError, "Invalid event name: #{event}" unless
|
254
|
-
|
328
|
+
raise ArgumentError, "Invalid event name: #{event}" unless self[:lifecycle_events].key?(event)
|
329
|
+
self[:lifecycle_events][event] << block
|
255
330
|
end
|
256
331
|
|
257
332
|
def self.strict_args!(mode = :raise)
|
258
|
-
|
333
|
+
self[:on_complex_arguments] = mode
|
259
334
|
end
|
260
335
|
|
261
|
-
# We are shutting down Sidekiq but what about
|
336
|
+
# We are shutting down Sidekiq but what about threads that
|
262
337
|
# are working on some long job? This error is
|
263
|
-
# raised in
|
338
|
+
# raised in jobs that have not finished within the hard
|
264
339
|
# timeout limit. This is needed to rollback db transactions,
|
265
340
|
# otherwise Ruby's Thread#kill will commit. See #377.
|
266
|
-
# DO NOT RESCUE THIS ERROR IN YOUR
|
341
|
+
# DO NOT RESCUE THIS ERROR IN YOUR JOBS
|
267
342
|
class Shutdown < Interrupt; end
|
268
343
|
end
|
269
344
|
|
data/sidekiq.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
22
22
|
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
23
|
}
|
24
24
|
|
25
|
-
gem.add_dependency "redis", ">= 4.
|
26
|
-
gem.add_dependency "connection_pool", ">= 2.2.
|
25
|
+
gem.add_dependency "redis", "<5", ">= 4.5.0"
|
26
|
+
gem.add_dependency "connection_pool", ">= 2.2.5"
|
27
27
|
gem.add_dependency "rack", "~> 2.0"
|
28
28
|
end
|
@@ -9,7 +9,9 @@ var ready = (callback) => {
|
|
9
9
|
else document.addEventListener("DOMContentLoaded", callback);
|
10
10
|
}
|
11
11
|
|
12
|
-
ready(
|
12
|
+
ready(addListeners)
|
13
|
+
|
14
|
+
function addListeners() {
|
13
15
|
document.querySelectorAll(".check_all").forEach(node => {
|
14
16
|
node.addEventListener("click", event => {
|
15
17
|
node.closest('table').querySelectorAll('input[type=checkbox]').forEach(inp => { inp.checked = !!node.checked; });
|
@@ -26,42 +28,48 @@ ready(() => {
|
|
26
28
|
})
|
27
29
|
|
28
30
|
document.querySelectorAll("[data-toggle]").forEach(node => {
|
29
|
-
node.addEventListener("click",
|
30
|
-
var targName = node.getAttribute("data-toggle");
|
31
|
-
var full = document.getElementById(targName + "_full");
|
32
|
-
if (full.style.display == "block") {
|
33
|
-
full.style.display = 'none';
|
34
|
-
} else {
|
35
|
-
full.style.display = 'block';
|
36
|
-
}
|
37
|
-
})
|
31
|
+
node.addEventListener("click", addDataToggleListeners)
|
38
32
|
})
|
39
33
|
|
40
34
|
updateFuzzyTimes();
|
35
|
+
setLivePollFromUrl();
|
41
36
|
|
42
37
|
var buttons = document.querySelectorAll(".live-poll");
|
43
38
|
if (buttons.length > 0) {
|
44
39
|
buttons.forEach(node => {
|
45
|
-
node.addEventListener("click",
|
46
|
-
if (localStorage.sidekiqLivePoll == "enabled") {
|
47
|
-
localStorage.sidekiqLivePoll = "disabled";
|
48
|
-
clearTimeout(livePollTimer);
|
49
|
-
livePollTimer = null;
|
50
|
-
} else {
|
51
|
-
localStorage.sidekiqLivePoll = "enabled";
|
52
|
-
livePollCallback();
|
53
|
-
}
|
54
|
-
|
55
|
-
updateLivePollButton();
|
56
|
-
})
|
40
|
+
node.addEventListener("click", addPollingListeners)
|
57
41
|
});
|
58
42
|
|
59
43
|
updateLivePollButton();
|
60
|
-
if (localStorage.sidekiqLivePoll == "enabled") {
|
44
|
+
if (localStorage.sidekiqLivePoll == "enabled" && !livePollTimer) {
|
61
45
|
scheduleLivePoll();
|
62
46
|
}
|
63
47
|
}
|
64
|
-
}
|
48
|
+
}
|
49
|
+
|
50
|
+
function addPollingListeners(_event) {
|
51
|
+
if (localStorage.sidekiqLivePoll == "enabled") {
|
52
|
+
localStorage.sidekiqLivePoll = "disabled";
|
53
|
+
clearTimeout(livePollTimer);
|
54
|
+
livePollTimer = null;
|
55
|
+
} else {
|
56
|
+
localStorage.sidekiqLivePoll = "enabled";
|
57
|
+
livePollCallback();
|
58
|
+
}
|
59
|
+
|
60
|
+
updateLivePollButton();
|
61
|
+
}
|
62
|
+
|
63
|
+
function addDataToggleListeners(event) {
|
64
|
+
var source = event.target || event.srcElement;
|
65
|
+
var targName = source.getAttribute("data-toggle");
|
66
|
+
var full = document.getElementById(targName);
|
67
|
+
if (full.style.display == "block") {
|
68
|
+
full.style.display = 'none';
|
69
|
+
} else {
|
70
|
+
full.style.display = 'block';
|
71
|
+
}
|
72
|
+
}
|
65
73
|
|
66
74
|
function updateFuzzyTimes() {
|
67
75
|
var locale = document.body.getAttribute("data-locale");
|
@@ -76,6 +84,14 @@ function updateFuzzyTimes() {
|
|
76
84
|
t.cancel();
|
77
85
|
}
|
78
86
|
|
87
|
+
function setLivePollFromUrl() {
|
88
|
+
var url_params = new URL(window.location.href).searchParams
|
89
|
+
|
90
|
+
if (url_params.get("poll") == "true") {
|
91
|
+
localStorage.sidekiqLivePoll = "enabled";
|
92
|
+
}
|
93
|
+
}
|
94
|
+
|
79
95
|
function updateLivePollButton() {
|
80
96
|
if (localStorage.sidekiqLivePoll == "enabled") {
|
81
97
|
document.querySelectorAll('.live-poll-stop').forEach(box => { box.style.display = "inline-block" })
|
@@ -89,7 +105,19 @@ function updateLivePollButton() {
|
|
89
105
|
function livePollCallback() {
|
90
106
|
clearTimeout(livePollTimer);
|
91
107
|
|
92
|
-
fetch(window.location.href)
|
108
|
+
fetch(window.location.href)
|
109
|
+
.then(checkResponse)
|
110
|
+
.then(resp => resp.text())
|
111
|
+
.then(replacePage)
|
112
|
+
.catch(showError)
|
113
|
+
.finally(scheduleLivePoll)
|
114
|
+
}
|
115
|
+
|
116
|
+
function checkResponse(resp) {
|
117
|
+
if (!resp.ok) {
|
118
|
+
throw response.error();
|
119
|
+
}
|
120
|
+
return resp
|
93
121
|
}
|
94
122
|
|
95
123
|
function scheduleLivePoll() {
|
@@ -107,5 +135,9 @@ function replacePage(text) {
|
|
107
135
|
var header_status = doc.querySelector('.status')
|
108
136
|
document.querySelector('.status').replaceWith(header_status)
|
109
137
|
|
110
|
-
|
138
|
+
addListeners();
|
139
|
+
}
|
140
|
+
|
141
|
+
function showError(error) {
|
142
|
+
console.error(error)
|
111
143
|
}
|