sidekiq 5.1.3 → 6.4.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +23 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +68 -62
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +344 -243
- data/lib/sidekiq/cli.rb +209 -221
- data/lib/sidekiq/client.rb +62 -64
- data/lib/sidekiq/delay.rb +7 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
- data/lib/sidekiq/fetch.rb +40 -32
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +35 -9
- data/lib/sidekiq/job_retry.rb +88 -68
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +170 -73
- data/lib/sidekiq/logger.rb +170 -0
- data/lib/sidekiq/manager.rb +18 -22
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +135 -83
- data/lib/sidekiq/rails.rb +42 -38
- data/lib/sidekiq/redis_connection.rb +49 -30
- data/lib/sidekiq/scheduled.rb +94 -31
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +40 -31
- data/lib/sidekiq/util.rb +57 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +109 -74
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +117 -93
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +61 -105
- data/lib/sidekiq/worker.rb +257 -99
- data/lib/sidekiq.rb +81 -46
- data/sidekiq.gemspec +23 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +66 -75
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +77 -231
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +9 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +7 -1
- data/web/locales/es.yml +21 -5
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +3 -17
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +4 -2
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +20 -10
- data/web/views/queues.erb +11 -3
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +39 -136
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -216
- data/Gemfile +0 -8
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -729
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/api.rb
CHANGED
@@ -1,10 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "zlib"
|
6
|
+
require "base64"
|
3
7
|
|
4
8
|
module Sidekiq
|
5
9
|
class Stats
|
6
10
|
def initialize
|
7
|
-
|
11
|
+
fetch_stats_fast!
|
8
12
|
end
|
9
13
|
|
10
14
|
def processed
|
@@ -47,56 +51,78 @@ module Sidekiq
|
|
47
51
|
Sidekiq::Stats::Queues.new.lengths
|
48
52
|
end
|
49
53
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
conn.smembers('queues')
|
54
|
+
# O(1) redis calls
|
55
|
+
def fetch_stats_fast!
|
56
|
+
pipe1_res = Sidekiq.redis { |conn|
|
57
|
+
conn.pipelined do |pipeline|
|
58
|
+
pipeline.get("stat:processed")
|
59
|
+
pipeline.get("stat:failed")
|
60
|
+
pipeline.zcard("schedule")
|
61
|
+
pipeline.zcard("retry")
|
62
|
+
pipeline.zcard("dead")
|
63
|
+
pipeline.scard("processes")
|
64
|
+
pipeline.lrange("queue:default", -1, -1)
|
62
65
|
end
|
63
|
-
|
66
|
+
}
|
64
67
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
68
|
+
default_queue_latency = if (entry = pipe1_res[6].first)
|
69
|
+
job = begin
|
70
|
+
Sidekiq.load_json(entry)
|
71
|
+
rescue
|
72
|
+
{}
|
69
73
|
end
|
74
|
+
now = Time.now.to_f
|
75
|
+
thence = job["enqueued_at"] || now
|
76
|
+
now - thence
|
77
|
+
else
|
78
|
+
0
|
70
79
|
end
|
71
80
|
|
72
|
-
s = pipe1_res[7].size
|
73
|
-
workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
|
74
|
-
enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
|
75
|
-
|
76
|
-
default_queue_latency = if (entry = pipe1_res[6].first)
|
77
|
-
job = Sidekiq.load_json(entry) rescue {}
|
78
|
-
now = Time.now.to_f
|
79
|
-
thence = job['enqueued_at'] || now
|
80
|
-
now - thence
|
81
|
-
else
|
82
|
-
0
|
83
|
-
end
|
84
81
|
@stats = {
|
85
|
-
processed:
|
86
|
-
failed:
|
87
|
-
scheduled_size:
|
88
|
-
retry_size:
|
89
|
-
dead_size:
|
90
|
-
processes_size:
|
91
|
-
|
92
|
-
default_queue_latency: default_queue_latency
|
93
|
-
|
94
|
-
|
82
|
+
processed: pipe1_res[0].to_i,
|
83
|
+
failed: pipe1_res[1].to_i,
|
84
|
+
scheduled_size: pipe1_res[2],
|
85
|
+
retry_size: pipe1_res[3],
|
86
|
+
dead_size: pipe1_res[4],
|
87
|
+
processes_size: pipe1_res[5],
|
88
|
+
|
89
|
+
default_queue_latency: default_queue_latency
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
# O(number of processes + number of queues) redis calls
|
94
|
+
def fetch_stats_slow!
|
95
|
+
processes = Sidekiq.redis { |conn|
|
96
|
+
conn.sscan_each("processes").to_a
|
95
97
|
}
|
98
|
+
|
99
|
+
queues = Sidekiq.redis { |conn|
|
100
|
+
conn.sscan_each("queues").to_a
|
101
|
+
}
|
102
|
+
|
103
|
+
pipe2_res = Sidekiq.redis { |conn|
|
104
|
+
conn.pipelined do |pipeline|
|
105
|
+
processes.each { |key| pipeline.hget(key, "busy") }
|
106
|
+
queues.each { |queue| pipeline.llen("queue:#{queue}") }
|
107
|
+
end
|
108
|
+
}
|
109
|
+
|
110
|
+
s = processes.size
|
111
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
112
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
113
|
+
|
114
|
+
@stats[:workers_size] = workers_size
|
115
|
+
@stats[:enqueued] = enqueued
|
116
|
+
@stats
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_stats!
|
120
|
+
fetch_stats_fast!
|
121
|
+
fetch_stats_slow!
|
96
122
|
end
|
97
123
|
|
98
124
|
def reset(*stats)
|
99
|
-
all
|
125
|
+
all = %w[failed processed]
|
100
126
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
101
127
|
|
102
128
|
mset_args = []
|
@@ -112,34 +138,31 @@ module Sidekiq
|
|
112
138
|
private
|
113
139
|
|
114
140
|
def stat(s)
|
115
|
-
@stats[s]
|
141
|
+
fetch_stats_slow! if @stats[s].nil?
|
142
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
116
143
|
end
|
117
144
|
|
118
145
|
class Queues
|
119
146
|
def lengths
|
120
147
|
Sidekiq.redis do |conn|
|
121
|
-
queues = conn.
|
148
|
+
queues = conn.sscan_each("queues").to_a
|
122
149
|
|
123
|
-
lengths = conn.pipelined
|
150
|
+
lengths = conn.pipelined { |pipeline|
|
124
151
|
queues.each do |queue|
|
125
|
-
|
152
|
+
pipeline.llen("queue:#{queue}")
|
126
153
|
end
|
127
|
-
|
128
|
-
|
129
|
-
i = 0
|
130
|
-
array_of_arrays = queues.inject({}) do |memo, queue|
|
131
|
-
memo[queue] = lengths[i]
|
132
|
-
i += 1
|
133
|
-
memo
|
134
|
-
end.sort_by { |_, size| size }
|
154
|
+
}
|
135
155
|
|
136
|
-
|
156
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
157
|
+
array_of_arrays.to_h
|
137
158
|
end
|
138
159
|
end
|
139
160
|
end
|
140
161
|
|
141
162
|
class History
|
142
163
|
def initialize(days_previous, start_date = nil)
|
164
|
+
# we only store five years of data in Redis
|
165
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
143
166
|
@days_previous = days_previous
|
144
167
|
@start_date = start_date || Time.now.utc.to_date
|
145
168
|
end
|
@@ -155,18 +178,12 @@ module Sidekiq
|
|
155
178
|
private
|
156
179
|
|
157
180
|
def date_stat_hash(stat)
|
158
|
-
i = 0
|
159
181
|
stat_hash = {}
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
datestr = date.strftime("%Y-%m-%d")
|
166
|
-
keys << "stat:#{stat}:#{datestr}"
|
167
|
-
dates << datestr
|
168
|
-
i += 1
|
169
|
-
end
|
182
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
183
|
+
date.strftime("%Y-%m-%d")
|
184
|
+
}
|
185
|
+
|
186
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
170
187
|
|
171
188
|
begin
|
172
189
|
Sidekiq.redis do |conn|
|
@@ -203,13 +220,13 @@ module Sidekiq
|
|
203
220
|
# Return all known queues within Redis.
|
204
221
|
#
|
205
222
|
def self.all
|
206
|
-
Sidekiq.redis { |c| c.
|
223
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
207
224
|
end
|
208
225
|
|
209
226
|
attr_reader :name
|
210
227
|
|
211
|
-
def initialize(name="default")
|
212
|
-
@name = name
|
228
|
+
def initialize(name = "default")
|
229
|
+
@name = name.to_s
|
213
230
|
@rname = "queue:#{name}"
|
214
231
|
end
|
215
232
|
|
@@ -228,13 +245,13 @@ module Sidekiq
|
|
228
245
|
#
|
229
246
|
# @return Float
|
230
247
|
def latency
|
231
|
-
entry = Sidekiq.redis
|
248
|
+
entry = Sidekiq.redis { |conn|
|
232
249
|
conn.lrange(@rname, -1, -1)
|
233
|
-
|
250
|
+
}.first
|
234
251
|
return 0 unless entry
|
235
252
|
job = Sidekiq.load_json(entry)
|
236
253
|
now = Time.now.to_f
|
237
|
-
thence = job[
|
254
|
+
thence = job["enqueued_at"] || now
|
238
255
|
now - thence
|
239
256
|
end
|
240
257
|
|
@@ -244,16 +261,16 @@ module Sidekiq
|
|
244
261
|
page = 0
|
245
262
|
page_size = 50
|
246
263
|
|
247
|
-
|
264
|
+
loop do
|
248
265
|
range_start = page * page_size - deleted_size
|
249
|
-
range_end
|
250
|
-
entries = Sidekiq.redis
|
266
|
+
range_end = range_start + page_size - 1
|
267
|
+
entries = Sidekiq.redis { |conn|
|
251
268
|
conn.lrange @rname, range_start, range_end
|
252
|
-
|
269
|
+
}
|
253
270
|
break if entries.empty?
|
254
271
|
page += 1
|
255
272
|
entries.each do |entry|
|
256
|
-
yield
|
273
|
+
yield JobRecord.new(entry, @name)
|
257
274
|
end
|
258
275
|
deleted_size = initial_size - size
|
259
276
|
end
|
@@ -263,16 +280,16 @@ module Sidekiq
|
|
263
280
|
# Find the job with the given JID within this queue.
|
264
281
|
#
|
265
282
|
# This is a slow, inefficient operation. Do not use under
|
266
|
-
# normal conditions.
|
283
|
+
# normal conditions.
|
267
284
|
def find_job(jid)
|
268
285
|
detect { |j| j.jid == jid }
|
269
286
|
end
|
270
287
|
|
271
288
|
def clear
|
272
289
|
Sidekiq.redis do |conn|
|
273
|
-
conn.multi do
|
274
|
-
|
275
|
-
|
290
|
+
conn.multi do |transaction|
|
291
|
+
transaction.unlink(@rname)
|
292
|
+
transaction.srem("queues", name)
|
276
293
|
end
|
277
294
|
end
|
278
295
|
end
|
@@ -284,17 +301,17 @@ module Sidekiq
|
|
284
301
|
# sorted set.
|
285
302
|
#
|
286
303
|
# The job should be considered immutable but may be
|
287
|
-
# removed from the queue via
|
304
|
+
# removed from the queue via JobRecord#delete.
|
288
305
|
#
|
289
|
-
class
|
306
|
+
class JobRecord
|
290
307
|
attr_reader :item
|
291
308
|
attr_reader :value
|
292
309
|
|
293
|
-
def initialize(item, queue_name=nil)
|
310
|
+
def initialize(item, queue_name = nil)
|
294
311
|
@args = nil
|
295
312
|
@value = item
|
296
313
|
@item = item.is_a?(Hash) ? item : parse(item)
|
297
|
-
@queue = queue_name || @item[
|
314
|
+
@queue = queue_name || @item["queue"]
|
298
315
|
end
|
299
316
|
|
300
317
|
def parse(item)
|
@@ -309,84 +326,105 @@ module Sidekiq
|
|
309
326
|
end
|
310
327
|
|
311
328
|
def klass
|
312
|
-
self[
|
329
|
+
self["class"]
|
313
330
|
end
|
314
331
|
|
315
332
|
def display_class
|
316
333
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
317
|
-
@klass ||=
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
334
|
+
@klass ||= self["display_class"] || begin
|
335
|
+
case klass
|
336
|
+
when /\ASidekiq::Extensions::Delayed/
|
337
|
+
safe_load(args[0], klass) do |target, method, _|
|
338
|
+
"#{target}.#{method}"
|
339
|
+
end
|
340
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
341
|
+
job_class = @item["wrapped"] || args[0]
|
342
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
343
|
+
# MailerClass#mailer_method
|
344
|
+
args[0]["arguments"][0..1].join("#")
|
345
|
+
else
|
346
|
+
job_class
|
347
|
+
end
|
348
|
+
else
|
349
|
+
klass
|
350
|
+
end
|
351
|
+
end
|
333
352
|
end
|
334
353
|
|
335
354
|
def display_args
|
336
355
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
337
356
|
@display_args ||= case klass
|
338
357
|
when /\ASidekiq::Extensions::Delayed/
|
339
|
-
safe_load(args[0], args) do |_, _, arg|
|
340
|
-
|
358
|
+
safe_load(args[0], args) do |_, _, arg, kwarg|
|
359
|
+
if !kwarg || kwarg.empty?
|
360
|
+
arg
|
361
|
+
else
|
362
|
+
[arg, kwarg]
|
363
|
+
end
|
341
364
|
end
|
342
365
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
343
|
-
job_args = self[
|
344
|
-
if
|
366
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
367
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
345
368
|
# remove MailerClass, mailer_method and 'deliver_now'
|
346
369
|
job_args.drop(3)
|
370
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
371
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
372
|
+
job_args.drop(3).first["args"]
|
347
373
|
else
|
348
374
|
job_args
|
349
375
|
end
|
350
376
|
else
|
351
|
-
if self[
|
377
|
+
if self["encrypt"]
|
352
378
|
# no point in showing 150+ bytes of random garbage
|
353
|
-
args[-1] =
|
379
|
+
args[-1] = "[encrypted data]"
|
354
380
|
end
|
355
381
|
args
|
356
|
-
|
382
|
+
end
|
357
383
|
end
|
358
384
|
|
359
385
|
def args
|
360
|
-
@args || @item[
|
386
|
+
@args || @item["args"]
|
361
387
|
end
|
362
388
|
|
363
389
|
def jid
|
364
|
-
self[
|
390
|
+
self["jid"]
|
365
391
|
end
|
366
392
|
|
367
393
|
def enqueued_at
|
368
|
-
self[
|
394
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
369
395
|
end
|
370
396
|
|
371
397
|
def created_at
|
372
|
-
Time.at(self[
|
398
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
373
399
|
end
|
374
400
|
|
375
|
-
def
|
376
|
-
|
401
|
+
def tags
|
402
|
+
self["tags"] || []
|
377
403
|
end
|
378
404
|
|
405
|
+
def error_backtrace
|
406
|
+
# Cache nil values
|
407
|
+
if defined?(@error_backtrace)
|
408
|
+
@error_backtrace
|
409
|
+
else
|
410
|
+
value = self["error_backtrace"]
|
411
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
412
|
+
end
|
413
|
+
end
|
414
|
+
|
415
|
+
attr_reader :queue
|
416
|
+
|
379
417
|
def latency
|
380
418
|
now = Time.now.to_f
|
381
|
-
now - (@item[
|
419
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
382
420
|
end
|
383
421
|
|
384
422
|
##
|
385
423
|
# Remove this job from the queue.
|
386
424
|
def delete
|
387
|
-
count = Sidekiq.redis
|
425
|
+
count = Sidekiq.redis { |conn|
|
388
426
|
conn.lrem("queue:#{@queue}", 1, @value)
|
389
|
-
|
427
|
+
}
|
390
428
|
count != 0
|
391
429
|
end
|
392
430
|
|
@@ -400,18 +438,33 @@ module Sidekiq
|
|
400
438
|
private
|
401
439
|
|
402
440
|
def safe_load(content, default)
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
441
|
+
yield(*YAML.load(content))
|
442
|
+
rescue => ex
|
443
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
444
|
+
# memory yet so the YAML can't be loaded.
|
445
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
446
|
+
default
|
447
|
+
end
|
448
|
+
|
449
|
+
def uncompress_backtrace(backtrace)
|
450
|
+
if backtrace.is_a?(Array)
|
451
|
+
# Handle old jobs with raw Array backtrace format
|
452
|
+
backtrace
|
453
|
+
else
|
454
|
+
decoded = Base64.decode64(backtrace)
|
455
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
456
|
+
begin
|
457
|
+
Sidekiq.load_json(uncompressed)
|
458
|
+
rescue
|
459
|
+
# Handle old jobs with marshalled backtrace format
|
460
|
+
# TODO Remove in 7.x
|
461
|
+
Marshal.load(uncompressed)
|
462
|
+
end
|
410
463
|
end
|
411
464
|
end
|
412
465
|
end
|
413
466
|
|
414
|
-
class SortedEntry <
|
467
|
+
class SortedEntry < JobRecord
|
415
468
|
attr_reader :score
|
416
469
|
attr_reader :parent
|
417
470
|
|
@@ -434,8 +487,9 @@ module Sidekiq
|
|
434
487
|
end
|
435
488
|
|
436
489
|
def reschedule(at)
|
437
|
-
|
438
|
-
|
490
|
+
Sidekiq.redis do |conn|
|
491
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
492
|
+
end
|
439
493
|
end
|
440
494
|
|
441
495
|
def add_to_queue
|
@@ -448,7 +502,7 @@ module Sidekiq
|
|
448
502
|
def retry
|
449
503
|
remove_job do |message|
|
450
504
|
msg = Sidekiq.load_json(message)
|
451
|
-
msg[
|
505
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
452
506
|
Sidekiq::Client.push(msg)
|
453
507
|
end
|
454
508
|
end
|
@@ -462,45 +516,44 @@ module Sidekiq
|
|
462
516
|
end
|
463
517
|
|
464
518
|
def error?
|
465
|
-
!!item[
|
519
|
+
!!item["error_class"]
|
466
520
|
end
|
467
521
|
|
468
522
|
private
|
469
523
|
|
470
524
|
def remove_job
|
471
525
|
Sidekiq.redis do |conn|
|
472
|
-
results = conn.multi
|
473
|
-
|
474
|
-
|
475
|
-
|
526
|
+
results = conn.multi { |transaction|
|
527
|
+
transaction.zrangebyscore(parent.name, score, score)
|
528
|
+
transaction.zremrangebyscore(parent.name, score, score)
|
529
|
+
}.first
|
476
530
|
|
477
531
|
if results.size == 1
|
478
532
|
yield results.first
|
479
533
|
else
|
480
534
|
# multiple jobs with the same score
|
481
535
|
# find the one with the right JID and push it
|
482
|
-
|
536
|
+
matched, nonmatched = results.partition { |message|
|
483
537
|
if message.index(jid)
|
484
538
|
msg = Sidekiq.load_json(message)
|
485
|
-
msg[
|
539
|
+
msg["jid"] == jid
|
486
540
|
else
|
487
541
|
false
|
488
542
|
end
|
489
|
-
|
543
|
+
}
|
490
544
|
|
491
|
-
msg =
|
545
|
+
msg = matched.first
|
492
546
|
yield msg if msg
|
493
547
|
|
494
548
|
# push the rest back onto the sorted set
|
495
|
-
conn.multi do
|
496
|
-
|
497
|
-
|
549
|
+
conn.multi do |transaction|
|
550
|
+
nonmatched.each do |message|
|
551
|
+
transaction.zadd(parent.name, score.to_f.to_s, message)
|
498
552
|
end
|
499
553
|
end
|
500
554
|
end
|
501
555
|
end
|
502
556
|
end
|
503
|
-
|
504
557
|
end
|
505
558
|
|
506
559
|
class SortedSet
|
@@ -517,16 +570,26 @@ module Sidekiq
|
|
517
570
|
Sidekiq.redis { |c| c.zcard(name) }
|
518
571
|
end
|
519
572
|
|
573
|
+
def scan(match, count = 100)
|
574
|
+
return to_enum(:scan, match, count) unless block_given?
|
575
|
+
|
576
|
+
match = "*#{match}*" unless match.include?("*")
|
577
|
+
Sidekiq.redis do |conn|
|
578
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
579
|
+
yield SortedEntry.new(self, score, entry)
|
580
|
+
end
|
581
|
+
end
|
582
|
+
end
|
583
|
+
|
520
584
|
def clear
|
521
585
|
Sidekiq.redis do |conn|
|
522
|
-
conn.
|
586
|
+
conn.unlink(name)
|
523
587
|
end
|
524
588
|
end
|
525
589
|
alias_method :💣, :clear
|
526
590
|
end
|
527
591
|
|
528
592
|
class JobSet < SortedSet
|
529
|
-
|
530
593
|
def schedule(timestamp, message)
|
531
594
|
Sidekiq.redis do |conn|
|
532
595
|
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
|
@@ -539,44 +602,55 @@ module Sidekiq
|
|
539
602
|
page = -1
|
540
603
|
page_size = 50
|
541
604
|
|
542
|
-
|
605
|
+
loop do
|
543
606
|
range_start = page * page_size + offset_size
|
544
|
-
range_end
|
545
|
-
elements = Sidekiq.redis
|
607
|
+
range_end = range_start + page_size - 1
|
608
|
+
elements = Sidekiq.redis { |conn|
|
546
609
|
conn.zrange name, range_start, range_end, with_scores: true
|
547
|
-
|
610
|
+
}
|
548
611
|
break if elements.empty?
|
549
612
|
page -= 1
|
550
|
-
elements.
|
613
|
+
elements.reverse_each do |element, score|
|
551
614
|
yield SortedEntry.new(self, score, element)
|
552
615
|
end
|
553
616
|
offset_size = initial_size - @_size
|
554
617
|
end
|
555
618
|
end
|
556
619
|
|
620
|
+
##
|
621
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
622
|
+
# optional second argument.
|
557
623
|
def fetch(score, jid = nil)
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
elements.inject([]) do |result, element|
|
563
|
-
entry = SortedEntry.new(self, score, element)
|
564
|
-
if jid
|
565
|
-
result << entry if entry.jid == jid
|
624
|
+
begin_score, end_score =
|
625
|
+
if score.is_a?(Range)
|
626
|
+
[score.first, score.last]
|
566
627
|
else
|
567
|
-
|
628
|
+
[score, score]
|
568
629
|
end
|
569
|
-
|
630
|
+
|
631
|
+
elements = Sidekiq.redis { |conn|
|
632
|
+
conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
|
633
|
+
}
|
634
|
+
|
635
|
+
elements.each_with_object([]) do |element, result|
|
636
|
+
data, job_score = element
|
637
|
+
entry = SortedEntry.new(self, job_score, data)
|
638
|
+
result << entry if jid.nil? || entry.jid == jid
|
570
639
|
end
|
571
640
|
end
|
572
641
|
|
573
642
|
##
|
574
643
|
# Find the job with the given JID within this sorted set.
|
575
|
-
#
|
576
|
-
# This is a slow, inefficient operation. Do not use under
|
577
|
-
# normal conditions. Sidekiq Pro contains a faster version.
|
644
|
+
# This is a slower O(n) operation. Do not use for app logic.
|
578
645
|
def find_job(jid)
|
579
|
-
|
646
|
+
Sidekiq.redis do |conn|
|
647
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
648
|
+
job = JSON.parse(entry)
|
649
|
+
matched = job["jid"] == jid
|
650
|
+
return SortedEntry.new(self, score, entry) if matched
|
651
|
+
end
|
652
|
+
end
|
653
|
+
nil
|
580
654
|
end
|
581
655
|
|
582
656
|
def delete_by_value(name, value)
|
@@ -591,13 +665,14 @@ module Sidekiq
|
|
591
665
|
Sidekiq.redis do |conn|
|
592
666
|
elements = conn.zrangebyscore(name, score, score)
|
593
667
|
elements.each do |element|
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
668
|
+
if element.index(jid)
|
669
|
+
message = Sidekiq.load_json(element)
|
670
|
+
if message["jid"] == jid
|
671
|
+
ret = conn.zrem(name, element)
|
672
|
+
@_size -= 1 if ret
|
673
|
+
break ret
|
674
|
+
end
|
599
675
|
end
|
600
|
-
false
|
601
676
|
end
|
602
677
|
end
|
603
678
|
end
|
@@ -619,7 +694,7 @@ module Sidekiq
|
|
619
694
|
# end.map(&:delete)
|
620
695
|
class ScheduledSet < JobSet
|
621
696
|
def initialize
|
622
|
-
super
|
697
|
+
super "schedule"
|
623
698
|
end
|
624
699
|
end
|
625
700
|
|
@@ -637,13 +712,15 @@ module Sidekiq
|
|
637
712
|
# end.map(&:delete)
|
638
713
|
class RetrySet < JobSet
|
639
714
|
def initialize
|
640
|
-
super
|
715
|
+
super "retry"
|
641
716
|
end
|
642
717
|
|
643
718
|
def retry_all
|
644
|
-
while size > 0
|
645
|
-
|
646
|
-
|
719
|
+
each(&:retry) while size > 0
|
720
|
+
end
|
721
|
+
|
722
|
+
def kill_all
|
723
|
+
each(&:kill) while size > 0
|
647
724
|
end
|
648
725
|
end
|
649
726
|
|
@@ -652,16 +729,16 @@ module Sidekiq
|
|
652
729
|
#
|
653
730
|
class DeadSet < JobSet
|
654
731
|
def initialize
|
655
|
-
super
|
732
|
+
super "dead"
|
656
733
|
end
|
657
734
|
|
658
|
-
def kill(message, opts={})
|
735
|
+
def kill(message, opts = {})
|
659
736
|
now = Time.now.to_f
|
660
737
|
Sidekiq.redis do |conn|
|
661
|
-
conn.multi do
|
662
|
-
|
663
|
-
|
664
|
-
|
738
|
+
conn.multi do |transaction|
|
739
|
+
transaction.zadd(name, now.to_s, message)
|
740
|
+
transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
741
|
+
transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
|
665
742
|
end
|
666
743
|
end
|
667
744
|
|
@@ -677,9 +754,7 @@ module Sidekiq
|
|
677
754
|
end
|
678
755
|
|
679
756
|
def retry_all
|
680
|
-
while size > 0
|
681
|
-
each(&:retry)
|
682
|
-
end
|
757
|
+
each(&:retry) while size > 0
|
683
758
|
end
|
684
759
|
|
685
760
|
def self.max_jobs
|
@@ -693,7 +768,7 @@ module Sidekiq
|
|
693
768
|
|
694
769
|
##
|
695
770
|
# Enumerates the set of Sidekiq processes which are actively working
|
696
|
-
# right now. Each process
|
771
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
697
772
|
# so this set should be relatively accurate, barring network partitions.
|
698
773
|
#
|
699
774
|
# Yields a Sidekiq::Process.
|
@@ -701,59 +776,60 @@ module Sidekiq
|
|
701
776
|
class ProcessSet
|
702
777
|
include Enumerable
|
703
778
|
|
704
|
-
def initialize(clean_plz=true)
|
705
|
-
|
779
|
+
def initialize(clean_plz = true)
|
780
|
+
cleanup if clean_plz
|
706
781
|
end
|
707
782
|
|
708
783
|
# Cleans up dead processes recorded in Redis.
|
709
784
|
# Returns the number of processes cleaned.
|
710
|
-
def
|
785
|
+
def cleanup
|
711
786
|
count = 0
|
712
787
|
Sidekiq.redis do |conn|
|
713
|
-
procs = conn.
|
714
|
-
heartbeats = conn.pipelined
|
788
|
+
procs = conn.sscan_each("processes").to_a.sort
|
789
|
+
heartbeats = conn.pipelined { |pipeline|
|
715
790
|
procs.each do |key|
|
716
|
-
|
791
|
+
pipeline.hget(key, "info")
|
717
792
|
end
|
718
|
-
|
793
|
+
}
|
719
794
|
|
720
795
|
# the hash named key has an expiry of 60 seconds.
|
721
796
|
# if it's not found, that means the process has not reported
|
722
797
|
# in to Redis and probably died.
|
723
|
-
to_prune =
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
count = conn.srem('processes', to_prune) unless to_prune.empty?
|
798
|
+
to_prune = procs.select.with_index { |proc, i|
|
799
|
+
heartbeats[i].nil?
|
800
|
+
}
|
801
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
728
802
|
end
|
729
803
|
count
|
730
804
|
end
|
731
805
|
|
732
806
|
def each
|
733
|
-
|
807
|
+
result = Sidekiq.redis { |conn|
|
808
|
+
procs = conn.sscan_each("processes").to_a.sort
|
734
809
|
|
735
|
-
Sidekiq.redis do |conn|
|
736
810
|
# We're making a tradeoff here between consuming more memory instead of
|
737
811
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
738
812
|
# you'll be happier this way
|
739
|
-
|
813
|
+
conn.pipelined do |pipeline|
|
740
814
|
procs.each do |key|
|
741
|
-
|
815
|
+
pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
742
816
|
end
|
743
817
|
end
|
818
|
+
}
|
744
819
|
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
820
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
821
|
+
# If a process is stopped between when we query Redis for `procs` and
|
822
|
+
# when we query for `result`, we will have an item in `result` that is
|
823
|
+
# composed of `nil` values.
|
824
|
+
next if info.nil?
|
750
825
|
|
751
|
-
|
752
|
-
|
753
|
-
|
826
|
+
hash = Sidekiq.load_json(info)
|
827
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
828
|
+
"beat" => at_s.to_f,
|
829
|
+
"quiet" => quiet,
|
830
|
+
"rss" => rss.to_i,
|
831
|
+
"rtt_us" => rtt.to_i))
|
754
832
|
end
|
755
|
-
|
756
|
-
nil
|
757
833
|
end
|
758
834
|
|
759
835
|
# This method is not guaranteed accurate since it does not prune the set
|
@@ -761,17 +837,29 @@ module Sidekiq
|
|
761
837
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
762
838
|
# 60 seconds.
|
763
839
|
def size
|
764
|
-
Sidekiq.redis { |conn| conn.scard(
|
840
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
765
841
|
end
|
766
842
|
|
843
|
+
# Total number of threads available to execute jobs.
|
844
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
845
|
+
# less than or equal to your licensed concurrency.
|
846
|
+
def total_concurrency
|
847
|
+
sum { |x| x["concurrency"].to_i }
|
848
|
+
end
|
849
|
+
|
850
|
+
def total_rss_in_kb
|
851
|
+
sum { |x| x["rss"].to_i }
|
852
|
+
end
|
853
|
+
alias_method :total_rss, :total_rss_in_kb
|
854
|
+
|
767
855
|
# Returns the identity of the current cluster leader or "" if no leader.
|
768
856
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
769
857
|
# or Sidekiq Pro.
|
770
858
|
def leader
|
771
859
|
@leader ||= begin
|
772
|
-
x = Sidekiq.redis {|c| c.get("dear-leader") }
|
860
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
773
861
|
# need a non-falsy value so we can memoize
|
774
|
-
x
|
862
|
+
x ||= ""
|
775
863
|
x
|
776
864
|
end
|
777
865
|
end
|
@@ -798,11 +886,11 @@ module Sidekiq
|
|
798
886
|
end
|
799
887
|
|
800
888
|
def tag
|
801
|
-
self[
|
889
|
+
self["tag"]
|
802
890
|
end
|
803
891
|
|
804
892
|
def labels
|
805
|
-
Array(self[
|
893
|
+
Array(self["labels"])
|
806
894
|
end
|
807
895
|
|
808
896
|
def [](key)
|
@@ -810,23 +898,27 @@ module Sidekiq
|
|
810
898
|
end
|
811
899
|
|
812
900
|
def identity
|
813
|
-
self[
|
901
|
+
self["identity"]
|
902
|
+
end
|
903
|
+
|
904
|
+
def queues
|
905
|
+
self["queues"]
|
814
906
|
end
|
815
907
|
|
816
908
|
def quiet!
|
817
|
-
signal(
|
909
|
+
signal("TSTP")
|
818
910
|
end
|
819
911
|
|
820
912
|
def stop!
|
821
|
-
signal(
|
913
|
+
signal("TERM")
|
822
914
|
end
|
823
915
|
|
824
916
|
def dump_threads
|
825
|
-
signal(
|
917
|
+
signal("TTIN")
|
826
918
|
end
|
827
919
|
|
828
920
|
def stopping?
|
829
|
-
self[
|
921
|
+
self["quiet"] == "true"
|
830
922
|
end
|
831
923
|
|
832
924
|
private
|
@@ -834,18 +926,17 @@ module Sidekiq
|
|
834
926
|
def signal(sig)
|
835
927
|
key = "#{identity}-signals"
|
836
928
|
Sidekiq.redis do |c|
|
837
|
-
c.multi do
|
838
|
-
|
839
|
-
|
929
|
+
c.multi do |transaction|
|
930
|
+
transaction.lpush(key, sig)
|
931
|
+
transaction.expire(key, 60)
|
840
932
|
end
|
841
933
|
end
|
842
934
|
end
|
843
|
-
|
844
935
|
end
|
845
936
|
|
846
937
|
##
|
847
|
-
#
|
848
|
-
#
|
938
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
939
|
+
# It tracks the process and thread working on each job.
|
849
940
|
#
|
850
941
|
# WARNING WARNING WARNING
|
851
942
|
#
|
@@ -853,33 +944,40 @@ module Sidekiq
|
|
853
944
|
# If you call #size => 5 and then expect #each to be
|
854
945
|
# called 5 times, you're going to have a bad time.
|
855
946
|
#
|
856
|
-
#
|
857
|
-
#
|
858
|
-
#
|
947
|
+
# works = Sidekiq::WorkSet.new
|
948
|
+
# works.size => 2
|
949
|
+
# works.each do |process_id, thread_id, work|
|
859
950
|
# # process_id is a unique identifier per Sidekiq process
|
860
951
|
# # thread_id is a unique identifier per thread
|
861
952
|
# # work is a Hash which looks like:
|
862
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
953
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
863
954
|
# # run_at is an epoch Integer.
|
864
955
|
# end
|
865
956
|
#
|
866
|
-
class
|
957
|
+
class WorkSet
|
867
958
|
include Enumerable
|
868
959
|
|
869
|
-
def each
|
960
|
+
def each(&block)
|
961
|
+
results = []
|
870
962
|
Sidekiq.redis do |conn|
|
871
|
-
procs = conn.
|
963
|
+
procs = conn.sscan_each("processes").to_a
|
872
964
|
procs.sort.each do |key|
|
873
|
-
valid, workers = conn.pipelined
|
874
|
-
|
875
|
-
|
876
|
-
|
965
|
+
valid, workers = conn.pipelined { |pipeline|
|
966
|
+
pipeline.exists?(key)
|
967
|
+
pipeline.hgetall("#{key}:workers")
|
968
|
+
}
|
877
969
|
next unless valid
|
878
970
|
workers.each_pair do |tid, json|
|
879
|
-
|
971
|
+
hsh = Sidekiq.load_json(json)
|
972
|
+
p = hsh["payload"]
|
973
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
974
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
975
|
+
results << [key, tid, hsh]
|
880
976
|
end
|
881
977
|
end
|
882
978
|
end
|
979
|
+
|
980
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
883
981
|
end
|
884
982
|
|
885
983
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -890,18 +988,21 @@ module Sidekiq
|
|
890
988
|
# which can easily get out of sync with crashy processes.
|
891
989
|
def size
|
892
990
|
Sidekiq.redis do |conn|
|
893
|
-
procs = conn.
|
991
|
+
procs = conn.sscan_each("processes").to_a
|
894
992
|
if procs.empty?
|
895
993
|
0
|
896
994
|
else
|
897
|
-
conn.pipelined
|
995
|
+
conn.pipelined { |pipeline|
|
898
996
|
procs.each do |key|
|
899
|
-
|
997
|
+
pipeline.hget(key, "busy")
|
900
998
|
end
|
901
|
-
|
999
|
+
}.sum(&:to_i)
|
902
1000
|
end
|
903
1001
|
end
|
904
1002
|
end
|
905
1003
|
end
|
906
|
-
|
1004
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1005
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1006
|
+
# WorkSet better describes the data.
|
1007
|
+
Workers = WorkSet
|
907
1008
|
end
|