sidekiq 6.4.1 → 7.0.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +172 -12
- data/README.md +41 -33
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +188 -114
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +275 -161
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +83 -88
- data/lib/sidekiq/client.rb +55 -43
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +270 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +21 -22
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +76 -54
- data/lib/sidekiq/job_util.rb +59 -19
- data/lib/sidekiq/launcher.rb +90 -82
- data/lib/sidekiq/logger.rb +6 -45
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +16 -17
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +11 -3
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +12 -10
- data/lib/sidekiq/redis_client_adapter.rb +115 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +65 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +40 -9
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +34 -20
- data/lib/sidekiq/web.rb +7 -14
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -207
- data/sidekiq.gemspec +20 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +66 -299
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +42 -26
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +71 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +63 -28
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -362
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/api.rb
CHANGED
@@ -3,9 +3,28 @@
|
|
3
3
|
require "sidekiq"
|
4
4
|
|
5
5
|
require "zlib"
|
6
|
+
require "set"
|
6
7
|
require "base64"
|
7
8
|
|
9
|
+
require "sidekiq/metrics/query"
|
10
|
+
|
11
|
+
#
|
12
|
+
# Sidekiq's Data API provides a Ruby object model on top
|
13
|
+
# of Sidekiq's runtime data in Redis. This API should never
|
14
|
+
# be used within application code for business logic.
|
15
|
+
#
|
16
|
+
# The Sidekiq server process never uses this API: all data
|
17
|
+
# manipulation is done directly for performance reasons to
|
18
|
+
# ensure we are using Redis as efficiently as possible at
|
19
|
+
# every callsite.
|
20
|
+
#
|
21
|
+
|
8
22
|
module Sidekiq
|
23
|
+
# Retrieve runtime statistics from Redis regarding
|
24
|
+
# this Sidekiq cluster.
|
25
|
+
#
|
26
|
+
# stat = Sidekiq::Stats.new
|
27
|
+
# stat.processed
|
9
28
|
class Stats
|
10
29
|
def initialize
|
11
30
|
fetch_stats_fast!
|
@@ -48,10 +67,22 @@ module Sidekiq
|
|
48
67
|
end
|
49
68
|
|
50
69
|
def queues
|
51
|
-
Sidekiq
|
70
|
+
Sidekiq.redis do |conn|
|
71
|
+
queues = conn.sscan("queues").to_a
|
72
|
+
|
73
|
+
lengths = conn.pipelined { |pipeline|
|
74
|
+
queues.each do |queue|
|
75
|
+
pipeline.llen("queue:#{queue}")
|
76
|
+
end
|
77
|
+
}
|
78
|
+
|
79
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
80
|
+
array_of_arrays.to_h
|
81
|
+
end
|
52
82
|
end
|
53
83
|
|
54
84
|
# O(1) redis calls
|
85
|
+
# @api private
|
55
86
|
def fetch_stats_fast!
|
56
87
|
pipe1_res = Sidekiq.redis { |conn|
|
57
88
|
conn.pipelined do |pipeline|
|
@@ -91,13 +122,14 @@ module Sidekiq
|
|
91
122
|
end
|
92
123
|
|
93
124
|
# O(number of processes + number of queues) redis calls
|
125
|
+
# @api private
|
94
126
|
def fetch_stats_slow!
|
95
127
|
processes = Sidekiq.redis { |conn|
|
96
|
-
conn.
|
128
|
+
conn.sscan("processes").to_a
|
97
129
|
}
|
98
130
|
|
99
131
|
queues = Sidekiq.redis { |conn|
|
100
|
-
conn.
|
132
|
+
conn.sscan("queues").to_a
|
101
133
|
}
|
102
134
|
|
103
135
|
pipe2_res = Sidekiq.redis { |conn|
|
@@ -109,18 +141,20 @@ module Sidekiq
|
|
109
141
|
|
110
142
|
s = processes.size
|
111
143
|
workers_size = pipe2_res[0...s].sum(&:to_i)
|
112
|
-
enqueued = pipe2_res[s
|
144
|
+
enqueued = pipe2_res[s..].sum(&:to_i)
|
113
145
|
|
114
146
|
@stats[:workers_size] = workers_size
|
115
147
|
@stats[:enqueued] = enqueued
|
116
148
|
@stats
|
117
149
|
end
|
118
150
|
|
151
|
+
# @api private
|
119
152
|
def fetch_stats!
|
120
153
|
fetch_stats_fast!
|
121
154
|
fetch_stats_slow!
|
122
155
|
end
|
123
156
|
|
157
|
+
# @api private
|
124
158
|
def reset(*stats)
|
125
159
|
all = %w[failed processed]
|
126
160
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
@@ -142,25 +176,8 @@ module Sidekiq
|
|
142
176
|
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
143
177
|
end
|
144
178
|
|
145
|
-
class Queues
|
146
|
-
def lengths
|
147
|
-
Sidekiq.redis do |conn|
|
148
|
-
queues = conn.sscan_each("queues").to_a
|
149
|
-
|
150
|
-
lengths = conn.pipelined { |pipeline|
|
151
|
-
queues.each do |queue|
|
152
|
-
pipeline.llen("queue:#{queue}")
|
153
|
-
end
|
154
|
-
}
|
155
|
-
|
156
|
-
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
157
|
-
array_of_arrays.to_h
|
158
|
-
end
|
159
|
-
end
|
160
|
-
end
|
161
|
-
|
162
179
|
class History
|
163
|
-
def initialize(days_previous, start_date = nil)
|
180
|
+
def initialize(days_previous, start_date = nil, pool: nil)
|
164
181
|
# we only store five years of data in Redis
|
165
182
|
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
166
183
|
@days_previous = days_previous
|
@@ -185,15 +202,10 @@ module Sidekiq
|
|
185
202
|
|
186
203
|
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
187
204
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
stat_hash[dates[idx]] = value ? value.to_i : 0
|
192
|
-
end
|
205
|
+
Sidekiq.redis do |conn|
|
206
|
+
conn.mget(keys).each_with_index do |value, idx|
|
207
|
+
stat_hash[dates[idx]] = value ? value.to_i : 0
|
193
208
|
end
|
194
|
-
rescue Redis::CommandError
|
195
|
-
# mget will trigger a CROSSSLOT error when run against a Cluster
|
196
|
-
# TODO Someone want to add Cluster support?
|
197
209
|
end
|
198
210
|
|
199
211
|
stat_hash
|
@@ -202,9 +214,10 @@ module Sidekiq
|
|
202
214
|
end
|
203
215
|
|
204
216
|
##
|
205
|
-
#
|
217
|
+
# Represents a queue within Sidekiq.
|
206
218
|
# Allows enumeration of all jobs within the queue
|
207
|
-
# and deletion of jobs.
|
219
|
+
# and deletion of jobs. NB: this queue data is real-time
|
220
|
+
# and is changing within Redis moment by moment.
|
208
221
|
#
|
209
222
|
# queue = Sidekiq::Queue.new("mailer")
|
210
223
|
# queue.each do |job|
|
@@ -212,29 +225,34 @@ module Sidekiq
|
|
212
225
|
# job.args # => [1, 2, 3]
|
213
226
|
# job.delete if job.jid == 'abcdef1234567890'
|
214
227
|
# end
|
215
|
-
#
|
216
228
|
class Queue
|
217
229
|
include Enumerable
|
218
230
|
|
219
231
|
##
|
220
|
-
#
|
232
|
+
# Fetch all known queues within Redis.
|
221
233
|
#
|
234
|
+
# @return [Array<Sidekiq::Queue>]
|
222
235
|
def self.all
|
223
|
-
Sidekiq.redis { |c| c.
|
236
|
+
Sidekiq.redis { |c| c.sscan("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
224
237
|
end
|
225
238
|
|
226
239
|
attr_reader :name
|
227
240
|
|
241
|
+
# @param name [String] the name of the queue
|
228
242
|
def initialize(name = "default")
|
229
243
|
@name = name.to_s
|
230
244
|
@rname = "queue:#{name}"
|
231
245
|
end
|
232
246
|
|
247
|
+
# The current size of the queue within Redis.
|
248
|
+
# This value is real-time and can change between calls.
|
249
|
+
#
|
250
|
+
# @return [Integer] the size
|
233
251
|
def size
|
234
252
|
Sidekiq.redis { |con| con.llen(@rname) }
|
235
253
|
end
|
236
254
|
|
237
|
-
#
|
255
|
+
# @return [Boolean] if the queue is currently paused
|
238
256
|
def paused?
|
239
257
|
false
|
240
258
|
end
|
@@ -243,7 +261,7 @@ module Sidekiq
|
|
243
261
|
# Calculates this queue's latency, the difference in seconds since the oldest
|
244
262
|
# job in the queue was enqueued.
|
245
263
|
#
|
246
|
-
# @return Float
|
264
|
+
# @return [Float] in seconds
|
247
265
|
def latency
|
248
266
|
entry = Sidekiq.redis { |conn|
|
249
267
|
conn.lrange(@rname, -1, -1)
|
@@ -279,34 +297,54 @@ module Sidekiq
|
|
279
297
|
##
|
280
298
|
# Find the job with the given JID within this queue.
|
281
299
|
#
|
282
|
-
# This is a slow, inefficient operation. Do not use under
|
300
|
+
# This is a *slow, inefficient* operation. Do not use under
|
283
301
|
# normal conditions.
|
302
|
+
#
|
303
|
+
# @param jid [String] the job_id to look for
|
304
|
+
# @return [Sidekiq::JobRecord]
|
305
|
+
# @return [nil] if not found
|
284
306
|
def find_job(jid)
|
285
307
|
detect { |j| j.jid == jid }
|
286
308
|
end
|
287
309
|
|
310
|
+
# delete all jobs within this queue
|
311
|
+
# @return [Boolean] true
|
288
312
|
def clear
|
289
313
|
Sidekiq.redis do |conn|
|
290
314
|
conn.multi do |transaction|
|
291
315
|
transaction.unlink(@rname)
|
292
|
-
transaction.srem("queues", name)
|
316
|
+
transaction.srem("queues", [name])
|
293
317
|
end
|
294
318
|
end
|
319
|
+
true
|
295
320
|
end
|
296
321
|
alias_method :💣, :clear
|
322
|
+
|
323
|
+
# :nodoc:
|
324
|
+
# @api private
|
325
|
+
def as_json(options = nil)
|
326
|
+
{name: name} # 5336
|
327
|
+
end
|
297
328
|
end
|
298
329
|
|
299
330
|
##
|
300
|
-
#
|
301
|
-
# sorted set.
|
331
|
+
# Represents a pending job within a Sidekiq queue.
|
302
332
|
#
|
303
333
|
# The job should be considered immutable but may be
|
304
334
|
# removed from the queue via JobRecord#delete.
|
305
|
-
#
|
306
335
|
class JobRecord
|
336
|
+
# the parsed Hash of job data
|
337
|
+
# @!attribute [r] Item
|
307
338
|
attr_reader :item
|
339
|
+
# the underlying String in Redis
|
340
|
+
# @!attribute [r] Value
|
308
341
|
attr_reader :value
|
342
|
+
# the queue associated with this job
|
343
|
+
# @!attribute [r] Queue
|
344
|
+
attr_reader :queue
|
309
345
|
|
346
|
+
# :nodoc:
|
347
|
+
# @api private
|
310
348
|
def initialize(item, queue_name = nil)
|
311
349
|
@args = nil
|
312
350
|
@value = item
|
@@ -314,6 +352,8 @@ module Sidekiq
|
|
314
352
|
@queue = queue_name || @item["queue"]
|
315
353
|
end
|
316
354
|
|
355
|
+
# :nodoc:
|
356
|
+
# @api private
|
317
357
|
def parse(item)
|
318
358
|
Sidekiq.load_json(item)
|
319
359
|
rescue JSON::ParserError
|
@@ -325,6 +365,8 @@ module Sidekiq
|
|
325
365
|
{}
|
326
366
|
end
|
327
367
|
|
368
|
+
# This is the job class which Sidekiq will execute. If using ActiveJob,
|
369
|
+
# this class will be the ActiveJob adapter class rather than a specific job.
|
328
370
|
def klass
|
329
371
|
self["class"]
|
330
372
|
end
|
@@ -332,12 +374,7 @@ module Sidekiq
|
|
332
374
|
def display_class
|
333
375
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
334
376
|
@klass ||= self["display_class"] || begin
|
335
|
-
|
336
|
-
when /\ASidekiq::Extensions::Delayed/
|
337
|
-
safe_load(args[0], klass) do |target, method, _|
|
338
|
-
"#{target}.#{method}"
|
339
|
-
end
|
340
|
-
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
377
|
+
if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
341
378
|
job_class = @item["wrapped"] || args[0]
|
342
379
|
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
343
380
|
# MailerClass#mailer_method
|
@@ -353,32 +390,23 @@ module Sidekiq
|
|
353
390
|
|
354
391
|
def display_args
|
355
392
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
356
|
-
@display_args ||=
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
else
|
374
|
-
job_args
|
375
|
-
end
|
376
|
-
else
|
377
|
-
if self["encrypt"]
|
378
|
-
# no point in showing 150+ bytes of random garbage
|
379
|
-
args[-1] = "[encrypted data]"
|
380
|
-
end
|
381
|
-
args
|
393
|
+
@display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
394
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
395
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
396
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
397
|
+
job_args.drop(3)
|
398
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
399
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
400
|
+
job_args.drop(3).first["args"]
|
401
|
+
else
|
402
|
+
job_args
|
403
|
+
end
|
404
|
+
else
|
405
|
+
if self["encrypt"]
|
406
|
+
# no point in showing 150+ bytes of random garbage
|
407
|
+
args[-1] = "[encrypted data]"
|
408
|
+
end
|
409
|
+
args
|
382
410
|
end
|
383
411
|
end
|
384
412
|
|
@@ -390,6 +418,10 @@ module Sidekiq
|
|
390
418
|
self["jid"]
|
391
419
|
end
|
392
420
|
|
421
|
+
def bid
|
422
|
+
self["bid"]
|
423
|
+
end
|
424
|
+
|
393
425
|
def enqueued_at
|
394
426
|
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
395
427
|
end
|
@@ -412,15 +444,12 @@ module Sidekiq
|
|
412
444
|
end
|
413
445
|
end
|
414
446
|
|
415
|
-
attr_reader :queue
|
416
|
-
|
417
447
|
def latency
|
418
448
|
now = Time.now.to_f
|
419
449
|
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
420
450
|
end
|
421
451
|
|
422
|
-
|
423
|
-
# Remove this job from the queue.
|
452
|
+
# Remove this job from the queue
|
424
453
|
def delete
|
425
454
|
count = Sidekiq.redis { |conn|
|
426
455
|
conn.lrem("queue:#{@queue}", 1, @value)
|
@@ -428,6 +457,7 @@ module Sidekiq
|
|
428
457
|
count != 0
|
429
458
|
end
|
430
459
|
|
460
|
+
# Access arbitrary attributes within the job hash
|
431
461
|
def [](name)
|
432
462
|
# nil will happen if the JSON fails to parse.
|
433
463
|
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
@@ -437,47 +467,35 @@ module Sidekiq
|
|
437
467
|
|
438
468
|
private
|
439
469
|
|
440
|
-
def safe_load(content, default)
|
441
|
-
yield(*YAML.load(content))
|
442
|
-
rescue => ex
|
443
|
-
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
444
|
-
# memory yet so the YAML can't be loaded.
|
445
|
-
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
446
|
-
default
|
447
|
-
end
|
448
|
-
|
449
470
|
def uncompress_backtrace(backtrace)
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
else
|
454
|
-
decoded = Base64.decode64(backtrace)
|
455
|
-
uncompressed = Zlib::Inflate.inflate(decoded)
|
456
|
-
begin
|
457
|
-
Sidekiq.load_json(uncompressed)
|
458
|
-
rescue
|
459
|
-
# Handle old jobs with marshalled backtrace format
|
460
|
-
# TODO Remove in 7.x
|
461
|
-
Marshal.load(uncompressed)
|
462
|
-
end
|
463
|
-
end
|
471
|
+
decoded = Base64.decode64(backtrace)
|
472
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
473
|
+
Sidekiq.load_json(uncompressed)
|
464
474
|
end
|
465
475
|
end
|
466
476
|
|
477
|
+
# Represents a job within a Redis sorted set where the score
|
478
|
+
# represents a timestamp associated with the job. This timestamp
|
479
|
+
# could be the scheduled time for it to run (e.g. scheduled set),
|
480
|
+
# or the expiration date after which the entry should be deleted (e.g. dead set).
|
467
481
|
class SortedEntry < JobRecord
|
468
482
|
attr_reader :score
|
469
483
|
attr_reader :parent
|
470
484
|
|
485
|
+
# :nodoc:
|
486
|
+
# @api private
|
471
487
|
def initialize(parent, score, item)
|
472
488
|
super(item)
|
473
|
-
@score = score
|
489
|
+
@score = Float(score)
|
474
490
|
@parent = parent
|
475
491
|
end
|
476
492
|
|
493
|
+
# The timestamp associated with this entry
|
477
494
|
def at
|
478
495
|
Time.at(score).utc
|
479
496
|
end
|
480
497
|
|
498
|
+
# remove this entry from the sorted set
|
481
499
|
def delete
|
482
500
|
if @value
|
483
501
|
@parent.delete_by_value(@parent.name, @value)
|
@@ -486,12 +504,17 @@ module Sidekiq
|
|
486
504
|
end
|
487
505
|
end
|
488
506
|
|
507
|
+
# Change the scheduled time for this job.
|
508
|
+
#
|
509
|
+
# @param at [Time] the new timestamp for this job
|
489
510
|
def reschedule(at)
|
490
511
|
Sidekiq.redis do |conn|
|
491
512
|
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
492
513
|
end
|
493
514
|
end
|
494
515
|
|
516
|
+
# Enqueue this job from the scheduled or dead set so it will
|
517
|
+
# be executed at some point in the near future.
|
495
518
|
def add_to_queue
|
496
519
|
remove_job do |message|
|
497
520
|
msg = Sidekiq.load_json(message)
|
@@ -499,6 +522,8 @@ module Sidekiq
|
|
499
522
|
end
|
500
523
|
end
|
501
524
|
|
525
|
+
# enqueue this job from the retry set so it will be executed
|
526
|
+
# at some point in the near future.
|
502
527
|
def retry
|
503
528
|
remove_job do |message|
|
504
529
|
msg = Sidekiq.load_json(message)
|
@@ -507,8 +532,7 @@ module Sidekiq
|
|
507
532
|
end
|
508
533
|
end
|
509
534
|
|
510
|
-
|
511
|
-
# Place job in the dead set
|
535
|
+
# Move this job from its current set into the Dead set.
|
512
536
|
def kill
|
513
537
|
remove_job do |message|
|
514
538
|
DeadSet.new.kill(message)
|
@@ -556,43 +580,69 @@ module Sidekiq
|
|
556
580
|
end
|
557
581
|
end
|
558
582
|
|
583
|
+
# Base class for all sorted sets within Sidekiq.
|
559
584
|
class SortedSet
|
560
585
|
include Enumerable
|
561
586
|
|
587
|
+
# Redis key of the set
|
588
|
+
# @!attribute [r] Name
|
562
589
|
attr_reader :name
|
563
590
|
|
591
|
+
# :nodoc:
|
592
|
+
# @api private
|
564
593
|
def initialize(name)
|
565
594
|
@name = name
|
566
595
|
@_size = size
|
567
596
|
end
|
568
597
|
|
598
|
+
# real-time size of the set, will change
|
569
599
|
def size
|
570
600
|
Sidekiq.redis { |c| c.zcard(name) }
|
571
601
|
end
|
572
602
|
|
603
|
+
# Scan through each element of the sorted set, yielding each to the supplied block.
|
604
|
+
# Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
|
605
|
+
#
|
606
|
+
# @param match [String] a snippet or regexp to filter matches.
|
607
|
+
# @param count [Integer] number of elements to retrieve at a time, default 100
|
608
|
+
# @yieldparam [Sidekiq::SortedEntry] each entry
|
573
609
|
def scan(match, count = 100)
|
574
610
|
return to_enum(:scan, match, count) unless block_given?
|
575
611
|
|
576
612
|
match = "*#{match}*" unless match.include?("*")
|
577
613
|
Sidekiq.redis do |conn|
|
578
|
-
conn.
|
614
|
+
conn.zscan(name, match: match, count: count) do |entry, score|
|
579
615
|
yield SortedEntry.new(self, score, entry)
|
580
616
|
end
|
581
617
|
end
|
582
618
|
end
|
583
619
|
|
620
|
+
# @return [Boolean] always true
|
584
621
|
def clear
|
585
622
|
Sidekiq.redis do |conn|
|
586
623
|
conn.unlink(name)
|
587
624
|
end
|
625
|
+
true
|
588
626
|
end
|
589
627
|
alias_method :💣, :clear
|
628
|
+
|
629
|
+
# :nodoc:
|
630
|
+
# @api private
|
631
|
+
def as_json(options = nil)
|
632
|
+
{name: name} # 5336
|
633
|
+
end
|
590
634
|
end
|
591
635
|
|
636
|
+
# Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
|
637
|
+
# Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
|
638
|
+
# e.g. Batches.
|
592
639
|
class JobSet < SortedSet
|
593
|
-
|
640
|
+
# Add a job with the associated timestamp to this set.
|
641
|
+
# @param timestamp [Time] the score for the job
|
642
|
+
# @param job [Hash] the job data
|
643
|
+
def schedule(timestamp, job)
|
594
644
|
Sidekiq.redis do |conn|
|
595
|
-
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(
|
645
|
+
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
|
596
646
|
end
|
597
647
|
end
|
598
648
|
|
@@ -606,7 +656,7 @@ module Sidekiq
|
|
606
656
|
range_start = page * page_size + offset_size
|
607
657
|
range_end = range_start + page_size - 1
|
608
658
|
elements = Sidekiq.redis { |conn|
|
609
|
-
conn.zrange name, range_start, range_end,
|
659
|
+
conn.zrange name, range_start, range_end, withscores: true
|
610
660
|
}
|
611
661
|
break if elements.empty?
|
612
662
|
page -= 1
|
@@ -620,6 +670,10 @@ module Sidekiq
|
|
620
670
|
##
|
621
671
|
# Fetch jobs that match a given time or Range. Job ID is an
|
622
672
|
# optional second argument.
|
673
|
+
#
|
674
|
+
# @param score [Time,Range] a specific timestamp or range
|
675
|
+
# @param jid [String, optional] find a specific JID within the score
|
676
|
+
# @return [Array<SortedEntry>] any results found, can be empty
|
623
677
|
def fetch(score, jid = nil)
|
624
678
|
begin_score, end_score =
|
625
679
|
if score.is_a?(Range)
|
@@ -629,7 +683,7 @@ module Sidekiq
|
|
629
683
|
end
|
630
684
|
|
631
685
|
elements = Sidekiq.redis { |conn|
|
632
|
-
conn.zrangebyscore(name, begin_score, end_score,
|
686
|
+
conn.zrangebyscore(name, begin_score, end_score, withscores: true)
|
633
687
|
}
|
634
688
|
|
635
689
|
elements.each_with_object([]) do |element, result|
|
@@ -641,11 +695,14 @@ module Sidekiq
|
|
641
695
|
|
642
696
|
##
|
643
697
|
# Find the job with the given JID within this sorted set.
|
644
|
-
# This is a
|
698
|
+
# *This is a slow O(n) operation*. Do not use for app logic.
|
699
|
+
#
|
700
|
+
# @param jid [String] the job identifier
|
701
|
+
# @return [SortedEntry] the record or nil
|
645
702
|
def find_job(jid)
|
646
703
|
Sidekiq.redis do |conn|
|
647
|
-
conn.
|
648
|
-
job =
|
704
|
+
conn.zscan(name, match: "*#{jid}*", count: 100) do |entry, score|
|
705
|
+
job = Sidekiq.load_json(entry)
|
649
706
|
matched = job["jid"] == jid
|
650
707
|
return SortedEntry.new(self, score, entry) if matched
|
651
708
|
end
|
@@ -653,6 +710,8 @@ module Sidekiq
|
|
653
710
|
nil
|
654
711
|
end
|
655
712
|
|
713
|
+
# :nodoc:
|
714
|
+
# @api private
|
656
715
|
def delete_by_value(name, value)
|
657
716
|
Sidekiq.redis do |conn|
|
658
717
|
ret = conn.zrem(name, value)
|
@@ -661,6 +720,8 @@ module Sidekiq
|
|
661
720
|
end
|
662
721
|
end
|
663
722
|
|
723
|
+
# :nodoc:
|
724
|
+
# @api private
|
664
725
|
def delete_by_jid(score, jid)
|
665
726
|
Sidekiq.redis do |conn|
|
666
727
|
elements = conn.zrangebyscore(name, score, score)
|
@@ -681,17 +742,13 @@ module Sidekiq
|
|
681
742
|
end
|
682
743
|
|
683
744
|
##
|
684
|
-
#
|
745
|
+
# The set of scheduled jobs within Sidekiq.
|
685
746
|
# Based on this, you can search/filter for jobs. Here's an
|
686
|
-
# example where I'm selecting
|
687
|
-
# and deleting them from the
|
747
|
+
# example where I'm selecting jobs based on some complex logic
|
748
|
+
# and deleting them from the scheduled set.
|
749
|
+
#
|
750
|
+
# See the API wiki page for usage notes and examples.
|
688
751
|
#
|
689
|
-
# r = Sidekiq::ScheduledSet.new
|
690
|
-
# r.select do |scheduled|
|
691
|
-
# scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
|
692
|
-
# scheduled.args[0] == 'User' &&
|
693
|
-
# scheduled.args[1] == 'setup_new_subscriber'
|
694
|
-
# end.map(&:delete)
|
695
752
|
class ScheduledSet < JobSet
|
696
753
|
def initialize
|
697
754
|
super "schedule"
|
@@ -699,46 +756,48 @@ module Sidekiq
|
|
699
756
|
end
|
700
757
|
|
701
758
|
##
|
702
|
-
#
|
759
|
+
# The set of retries within Sidekiq.
|
703
760
|
# Based on this, you can search/filter for jobs. Here's an
|
704
761
|
# example where I'm selecting all jobs of a certain type
|
705
762
|
# and deleting them from the retry queue.
|
706
763
|
#
|
707
|
-
#
|
708
|
-
#
|
709
|
-
# retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
|
710
|
-
# retri.args[0] == 'User' &&
|
711
|
-
# retri.args[1] == 'setup_new_subscriber'
|
712
|
-
# end.map(&:delete)
|
764
|
+
# See the API wiki page for usage notes and examples.
|
765
|
+
#
|
713
766
|
class RetrySet < JobSet
|
714
767
|
def initialize
|
715
768
|
super "retry"
|
716
769
|
end
|
717
770
|
|
771
|
+
# Enqueues all jobs pending within the retry set.
|
718
772
|
def retry_all
|
719
773
|
each(&:retry) while size > 0
|
720
774
|
end
|
721
775
|
|
776
|
+
# Kills all jobs pending within the retry set.
|
722
777
|
def kill_all
|
723
778
|
each(&:kill) while size > 0
|
724
779
|
end
|
725
780
|
end
|
726
781
|
|
727
782
|
##
|
728
|
-
#
|
783
|
+
# The set of dead jobs within Sidekiq. Dead jobs have failed all of
|
784
|
+
# their retries and are helding in this set pending some sort of manual
|
785
|
+
# fix. They will be removed after 6 months (dead_timeout) if not.
|
729
786
|
#
|
730
787
|
class DeadSet < JobSet
|
731
788
|
def initialize
|
732
789
|
super "dead"
|
733
790
|
end
|
734
791
|
|
792
|
+
# Add the given job to the Dead set.
|
793
|
+
# @param message [String] the job data as JSON
|
735
794
|
def kill(message, opts = {})
|
736
795
|
now = Time.now.to_f
|
737
796
|
Sidekiq.redis do |conn|
|
738
797
|
conn.multi do |transaction|
|
739
798
|
transaction.zadd(name, now.to_s, message)
|
740
|
-
transaction.zremrangebyscore(name, "-inf", now -
|
741
|
-
transaction.zremrangebyrank(name, 0, -
|
799
|
+
transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
|
800
|
+
transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
|
742
801
|
end
|
743
802
|
end
|
744
803
|
|
@@ -746,24 +805,17 @@ module Sidekiq
|
|
746
805
|
job = Sidekiq.load_json(message)
|
747
806
|
r = RuntimeError.new("Job killed by API")
|
748
807
|
r.set_backtrace(caller)
|
749
|
-
Sidekiq.death_handlers.each do |handle|
|
808
|
+
Sidekiq.default_configuration.death_handlers.each do |handle|
|
750
809
|
handle.call(job, r)
|
751
810
|
end
|
752
811
|
end
|
753
812
|
true
|
754
813
|
end
|
755
814
|
|
815
|
+
# Enqueue all dead jobs
|
756
816
|
def retry_all
|
757
817
|
each(&:retry) while size > 0
|
758
818
|
end
|
759
|
-
|
760
|
-
def self.max_jobs
|
761
|
-
Sidekiq.options[:dead_max_jobs]
|
762
|
-
end
|
763
|
-
|
764
|
-
def self.timeout
|
765
|
-
Sidekiq.options[:dead_timeout_in_seconds]
|
766
|
-
end
|
767
819
|
end
|
768
820
|
|
769
821
|
##
|
@@ -771,21 +823,46 @@ module Sidekiq
|
|
771
823
|
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
772
824
|
# so this set should be relatively accurate, barring network partitions.
|
773
825
|
#
|
774
|
-
#
|
826
|
+
# @yieldparam [Sidekiq::Process]
|
775
827
|
#
|
776
828
|
class ProcessSet
|
777
829
|
include Enumerable
|
778
830
|
|
831
|
+
def self.[](identity)
|
832
|
+
exists, (info, busy, beat, quiet, rss, rtt_us) = Sidekiq.redis { |conn|
|
833
|
+
conn.multi { |transaction|
|
834
|
+
transaction.sismember("processes", identity)
|
835
|
+
transaction.hmget(identity, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
836
|
+
}
|
837
|
+
}
|
838
|
+
|
839
|
+
return nil if exists == 0 || info.nil?
|
840
|
+
|
841
|
+
hash = Sidekiq.load_json(info)
|
842
|
+
Process.new(hash.merge("busy" => busy.to_i,
|
843
|
+
"beat" => beat.to_f,
|
844
|
+
"quiet" => quiet,
|
845
|
+
"rss" => rss.to_i,
|
846
|
+
"rtt_us" => rtt_us.to_i))
|
847
|
+
end
|
848
|
+
|
849
|
+
# :nodoc:
|
850
|
+
# @api private
|
779
851
|
def initialize(clean_plz = true)
|
780
852
|
cleanup if clean_plz
|
781
853
|
end
|
782
854
|
|
783
855
|
# Cleans up dead processes recorded in Redis.
|
784
856
|
# Returns the number of processes cleaned.
|
857
|
+
# :nodoc:
|
858
|
+
# @api private
|
785
859
|
def cleanup
|
860
|
+
# dont run cleanup more than once per minute
|
861
|
+
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
|
862
|
+
|
786
863
|
count = 0
|
787
864
|
Sidekiq.redis do |conn|
|
788
|
-
procs = conn.
|
865
|
+
procs = conn.sscan("processes").to_a
|
789
866
|
heartbeats = conn.pipelined { |pipeline|
|
790
867
|
procs.each do |key|
|
791
868
|
pipeline.hget(key, "info")
|
@@ -805,7 +882,7 @@ module Sidekiq
|
|
805
882
|
|
806
883
|
def each
|
807
884
|
result = Sidekiq.redis { |conn|
|
808
|
-
procs = conn.
|
885
|
+
procs = conn.sscan("processes").to_a.sort
|
809
886
|
|
810
887
|
# We're making a tradeoff here between consuming more memory instead of
|
811
888
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
@@ -817,7 +894,7 @@ module Sidekiq
|
|
817
894
|
end
|
818
895
|
}
|
819
896
|
|
820
|
-
result.each do |info, busy,
|
897
|
+
result.each do |info, busy, beat, quiet, rss, rtt_us|
|
821
898
|
# If a process is stopped between when we query Redis for `procs` and
|
822
899
|
# when we query for `result`, we will have an item in `result` that is
|
823
900
|
# composed of `nil` values.
|
@@ -825,10 +902,10 @@ module Sidekiq
|
|
825
902
|
|
826
903
|
hash = Sidekiq.load_json(info)
|
827
904
|
yield Process.new(hash.merge("busy" => busy.to_i,
|
828
|
-
"beat" =>
|
905
|
+
"beat" => beat.to_f,
|
829
906
|
"quiet" => quiet,
|
830
907
|
"rss" => rss.to_i,
|
831
|
-
"rtt_us" =>
|
908
|
+
"rtt_us" => rtt_us.to_i))
|
832
909
|
end
|
833
910
|
end
|
834
911
|
|
@@ -836,6 +913,7 @@ module Sidekiq
|
|
836
913
|
# based on current heartbeat. #each does that and ensures the set only
|
837
914
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
838
915
|
# 60 seconds.
|
916
|
+
# @return [Integer] current number of registered Sidekiq processes
|
839
917
|
def size
|
840
918
|
Sidekiq.redis { |conn| conn.scard("processes") }
|
841
919
|
end
|
@@ -843,10 +921,12 @@ module Sidekiq
|
|
843
921
|
# Total number of threads available to execute jobs.
|
844
922
|
# For Sidekiq Enterprise customers this number (in production) must be
|
845
923
|
# less than or equal to your licensed concurrency.
|
924
|
+
# @return [Integer] the sum of process concurrency
|
846
925
|
def total_concurrency
|
847
926
|
sum { |x| x["concurrency"].to_i }
|
848
927
|
end
|
849
928
|
|
929
|
+
# @return [Integer] total amount of RSS memory consumed by Sidekiq processes
|
850
930
|
def total_rss_in_kb
|
851
931
|
sum { |x| x["rss"].to_i }
|
852
932
|
end
|
@@ -855,6 +935,8 @@ module Sidekiq
|
|
855
935
|
# Returns the identity of the current cluster leader or "" if no leader.
|
856
936
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
857
937
|
# or Sidekiq Pro.
|
938
|
+
# @return [String] Identity of cluster leader
|
939
|
+
# @return [String] empty string if no leader
|
858
940
|
def leader
|
859
941
|
@leader ||= begin
|
860
942
|
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
@@ -879,8 +961,11 @@ module Sidekiq
|
|
879
961
|
# 'busy' => 10,
|
880
962
|
# 'beat' => <last heartbeat>,
|
881
963
|
# 'identity' => <unique string identifying the process>,
|
964
|
+
# 'embedded' => true,
|
882
965
|
# }
|
883
966
|
class Process
|
967
|
+
# :nodoc:
|
968
|
+
# @api private
|
884
969
|
def initialize(hash)
|
885
970
|
@attribs = hash
|
886
971
|
end
|
@@ -890,7 +975,7 @@ module Sidekiq
|
|
890
975
|
end
|
891
976
|
|
892
977
|
def labels
|
893
|
-
|
978
|
+
self["labels"].to_a
|
894
979
|
end
|
895
980
|
|
896
981
|
def [](key)
|
@@ -905,18 +990,47 @@ module Sidekiq
|
|
905
990
|
self["queues"]
|
906
991
|
end
|
907
992
|
|
993
|
+
def weights
|
994
|
+
self["weights"]
|
995
|
+
end
|
996
|
+
|
997
|
+
def version
|
998
|
+
self["version"]
|
999
|
+
end
|
1000
|
+
|
1001
|
+
def embedded?
|
1002
|
+
self["embedded"]
|
1003
|
+
end
|
1004
|
+
|
1005
|
+
# Signal this process to stop processing new jobs.
|
1006
|
+
# It will continue to execute jobs it has already fetched.
|
1007
|
+
# This method is *asynchronous* and it can take 5-10
|
1008
|
+
# seconds for the process to quiet.
|
908
1009
|
def quiet!
|
1010
|
+
raise "Can't quiet an embedded process" if embedded?
|
1011
|
+
|
909
1012
|
signal("TSTP")
|
910
1013
|
end
|
911
1014
|
|
1015
|
+
# Signal this process to shutdown.
|
1016
|
+
# It will shutdown within its configured :timeout value, default 25 seconds.
|
1017
|
+
# This method is *asynchronous* and it can take 5-10
|
1018
|
+
# seconds for the process to start shutting down.
|
912
1019
|
def stop!
|
1020
|
+
raise "Can't stop an embedded process" if embedded?
|
1021
|
+
|
913
1022
|
signal("TERM")
|
914
1023
|
end
|
915
1024
|
|
1025
|
+
# Signal this process to log backtraces for all threads.
|
1026
|
+
# Useful if you have a frozen or deadlocked process which is
|
1027
|
+
# still sending a heartbeat.
|
1028
|
+
# This method is *asynchronous* and it can take 5-10 seconds.
|
916
1029
|
def dump_threads
|
917
1030
|
signal("TTIN")
|
918
1031
|
end
|
919
1032
|
|
1033
|
+
# @return [Boolean] true if this process is quiet or shutting down
|
920
1034
|
def stopping?
|
921
1035
|
self["quiet"] == "true"
|
922
1036
|
end
|
@@ -959,24 +1073,24 @@ module Sidekiq
|
|
959
1073
|
|
960
1074
|
def each(&block)
|
961
1075
|
results = []
|
1076
|
+
procs = nil
|
1077
|
+
all_works = nil
|
1078
|
+
|
962
1079
|
Sidekiq.redis do |conn|
|
963
|
-
procs = conn.
|
964
|
-
|
965
|
-
|
966
|
-
pipeline.
|
967
|
-
pipeline.hgetall("#{key}:workers")
|
968
|
-
}
|
969
|
-
next unless valid
|
970
|
-
workers.each_pair do |tid, json|
|
971
|
-
hsh = Sidekiq.load_json(json)
|
972
|
-
p = hsh["payload"]
|
973
|
-
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
974
|
-
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
975
|
-
results << [key, tid, hsh]
|
1080
|
+
procs = conn.sscan("processes").to_a.sort
|
1081
|
+
all_works = conn.pipelined do |pipeline|
|
1082
|
+
procs.each do |key|
|
1083
|
+
pipeline.hgetall("#{key}:work")
|
976
1084
|
end
|
977
1085
|
end
|
978
1086
|
end
|
979
1087
|
|
1088
|
+
procs.zip(all_works).each do |key, workers|
|
1089
|
+
workers.each_pair do |tid, json|
|
1090
|
+
results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
|
1091
|
+
end
|
1092
|
+
end
|
1093
|
+
|
980
1094
|
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
981
1095
|
end
|
982
1096
|
|
@@ -988,7 +1102,7 @@ module Sidekiq
|
|
988
1102
|
# which can easily get out of sync with crashy processes.
|
989
1103
|
def size
|
990
1104
|
Sidekiq.redis do |conn|
|
991
|
-
procs = conn.
|
1105
|
+
procs = conn.sscan("processes").to_a
|
992
1106
|
if procs.empty?
|
993
1107
|
0
|
994
1108
|
else
|