sidekiq 4.2.4 → 6.2.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +445 -0
- data/LICENSE +1 -1
- data/README.md +21 -34
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +2 -2
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +347 -213
- data/lib/sidekiq/cli.rb +221 -212
- data/lib/sidekiq/client.rb +75 -52
- data/lib/sidekiq/delay.rb +41 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +10 -4
- data/lib/sidekiq/fetch.rb +38 -31
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +263 -0
- data/lib/sidekiq/launcher.rb +169 -70
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +17 -20
- data/lib/sidekiq/middleware/chain.rb +15 -5
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -70
- data/lib/sidekiq/rails.rb +30 -73
- data/lib/sidekiq/redis_connection.rb +67 -20
- data/lib/sidekiq/scheduled.rb +61 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +54 -26
- data/lib/sidekiq/util.rb +48 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +112 -89
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +153 -73
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +64 -109
- data/lib/sidekiq/worker.rb +164 -41
- data/lib/sidekiq.rb +86 -60
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/javascripts/dashboard.js +34 -38
- data/web/assets/stylesheets/application-dark.css +160 -0
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +402 -12
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +81 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +4 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +3 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +9 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +57 -19
- data/web/views/dashboard.erb +3 -3
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +13 -2
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +22 -12
- data/web/views/queues.erb +13 -3
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +42 -194
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -585
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/api.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "zlib"
|
6
|
+
require "base64"
|
4
7
|
|
5
8
|
module Sidekiq
|
6
9
|
class Stats
|
@@ -49,52 +52,65 @@ module Sidekiq
|
|
49
52
|
end
|
50
53
|
|
51
54
|
def fetch_stats!
|
52
|
-
pipe1_res = Sidekiq.redis
|
55
|
+
pipe1_res = Sidekiq.redis { |conn|
|
53
56
|
conn.pipelined do
|
54
|
-
conn.get(
|
55
|
-
conn.get(
|
56
|
-
conn.zcard(
|
57
|
-
conn.zcard(
|
58
|
-
conn.zcard(
|
59
|
-
conn.scard(
|
60
|
-
conn.lrange(
|
61
|
-
conn.smembers('processes'.freeze)
|
62
|
-
conn.smembers('queues'.freeze)
|
57
|
+
conn.get("stat:processed")
|
58
|
+
conn.get("stat:failed")
|
59
|
+
conn.zcard("schedule")
|
60
|
+
conn.zcard("retry")
|
61
|
+
conn.zcard("dead")
|
62
|
+
conn.scard("processes")
|
63
|
+
conn.lrange("queue:default", -1, -1)
|
63
64
|
end
|
64
|
-
|
65
|
+
}
|
66
|
+
|
67
|
+
processes = Sidekiq.redis { |conn|
|
68
|
+
conn.sscan_each("processes").to_a
|
69
|
+
}
|
70
|
+
|
71
|
+
queues = Sidekiq.redis { |conn|
|
72
|
+
conn.sscan_each("queues").to_a
|
73
|
+
}
|
65
74
|
|
66
|
-
pipe2_res = Sidekiq.redis
|
75
|
+
pipe2_res = Sidekiq.redis { |conn|
|
67
76
|
conn.pipelined do
|
68
|
-
|
69
|
-
|
77
|
+
processes.each { |key| conn.hget(key, "busy") }
|
78
|
+
queues.each { |queue| conn.llen("queue:#{queue}") }
|
70
79
|
end
|
71
|
-
|
80
|
+
}
|
72
81
|
|
73
|
-
s =
|
74
|
-
workers_size = pipe2_res[0...s].
|
75
|
-
enqueued
|
82
|
+
s = processes.size
|
83
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
84
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
76
85
|
|
77
86
|
default_queue_latency = if (entry = pipe1_res[6].first)
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
87
|
+
job = begin
|
88
|
+
Sidekiq.load_json(entry)
|
89
|
+
rescue
|
90
|
+
{}
|
91
|
+
end
|
92
|
+
now = Time.now.to_f
|
93
|
+
thence = job["enqueued_at"] || now
|
94
|
+
now - thence
|
95
|
+
else
|
96
|
+
0
|
97
|
+
end
|
82
98
|
@stats = {
|
83
|
-
processed:
|
84
|
-
failed:
|
85
|
-
scheduled_size:
|
86
|
-
retry_size:
|
87
|
-
dead_size:
|
88
|
-
processes_size:
|
99
|
+
processed: pipe1_res[0].to_i,
|
100
|
+
failed: pipe1_res[1].to_i,
|
101
|
+
scheduled_size: pipe1_res[2],
|
102
|
+
retry_size: pipe1_res[3],
|
103
|
+
dead_size: pipe1_res[4],
|
104
|
+
processes_size: pipe1_res[5],
|
89
105
|
|
90
106
|
default_queue_latency: default_queue_latency,
|
91
|
-
workers_size:
|
92
|
-
enqueued:
|
107
|
+
workers_size: workers_size,
|
108
|
+
enqueued: enqueued
|
93
109
|
}
|
94
110
|
end
|
95
111
|
|
96
112
|
def reset(*stats)
|
97
|
-
all
|
113
|
+
all = %w[failed processed]
|
98
114
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
99
115
|
|
100
116
|
mset_args = []
|
@@ -116,22 +132,16 @@ module Sidekiq
|
|
116
132
|
class Queues
|
117
133
|
def lengths
|
118
134
|
Sidekiq.redis do |conn|
|
119
|
-
queues = conn.
|
135
|
+
queues = conn.sscan_each("queues").to_a
|
120
136
|
|
121
|
-
lengths = conn.pipelined
|
137
|
+
lengths = conn.pipelined {
|
122
138
|
queues.each do |queue|
|
123
139
|
conn.llen("queue:#{queue}")
|
124
140
|
end
|
125
|
-
|
141
|
+
}
|
126
142
|
|
127
|
-
|
128
|
-
array_of_arrays
|
129
|
-
memo[queue] = lengths[i]
|
130
|
-
i += 1
|
131
|
-
memo
|
132
|
-
end.sort_by { |_, size| size }
|
133
|
-
|
134
|
-
Hash[array_of_arrays.reverse]
|
143
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
144
|
+
Hash[array_of_arrays]
|
135
145
|
end
|
136
146
|
end
|
137
147
|
end
|
@@ -143,33 +153,32 @@ module Sidekiq
|
|
143
153
|
end
|
144
154
|
|
145
155
|
def processed
|
146
|
-
date_stat_hash("processed")
|
156
|
+
@processed ||= date_stat_hash("processed")
|
147
157
|
end
|
148
158
|
|
149
159
|
def failed
|
150
|
-
date_stat_hash("failed")
|
160
|
+
@failed ||= date_stat_hash("failed")
|
151
161
|
end
|
152
162
|
|
153
163
|
private
|
154
164
|
|
155
165
|
def date_stat_hash(stat)
|
156
|
-
i = 0
|
157
166
|
stat_hash = {}
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
while i < @days_previous
|
162
|
-
date = @start_date - i
|
163
|
-
datestr = date.strftime("%Y-%m-%d".freeze)
|
164
|
-
keys << "stat:#{stat}:#{datestr}"
|
165
|
-
dates << datestr
|
166
|
-
i += 1
|
167
|
-
end
|
167
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
168
|
+
date.strftime("%Y-%m-%d")
|
169
|
+
}
|
168
170
|
|
169
|
-
|
170
|
-
|
171
|
-
|
171
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
172
|
+
|
173
|
+
begin
|
174
|
+
Sidekiq.redis do |conn|
|
175
|
+
conn.mget(keys).each_with_index do |value, idx|
|
176
|
+
stat_hash[dates[idx]] = value ? value.to_i : 0
|
177
|
+
end
|
172
178
|
end
|
179
|
+
rescue Redis::CommandError
|
180
|
+
# mget will trigger a CROSSSLOT error when run against a Cluster
|
181
|
+
# TODO Someone want to add Cluster support?
|
173
182
|
end
|
174
183
|
|
175
184
|
stat_hash
|
@@ -196,13 +205,13 @@ module Sidekiq
|
|
196
205
|
# Return all known queues within Redis.
|
197
206
|
#
|
198
207
|
def self.all
|
199
|
-
Sidekiq.redis { |c| c.
|
208
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
200
209
|
end
|
201
210
|
|
202
211
|
attr_reader :name
|
203
212
|
|
204
|
-
def initialize(name="default")
|
205
|
-
@name = name
|
213
|
+
def initialize(name = "default")
|
214
|
+
@name = name.to_s
|
206
215
|
@rname = "queue:#{name}"
|
207
216
|
end
|
208
217
|
|
@@ -221,11 +230,14 @@ module Sidekiq
|
|
221
230
|
#
|
222
231
|
# @return Float
|
223
232
|
def latency
|
224
|
-
entry = Sidekiq.redis
|
233
|
+
entry = Sidekiq.redis { |conn|
|
225
234
|
conn.lrange(@rname, -1, -1)
|
226
|
-
|
235
|
+
}.first
|
227
236
|
return 0 unless entry
|
228
|
-
|
237
|
+
job = Sidekiq.load_json(entry)
|
238
|
+
now = Time.now.to_f
|
239
|
+
thence = job["enqueued_at"] || now
|
240
|
+
now - thence
|
229
241
|
end
|
230
242
|
|
231
243
|
def each
|
@@ -234,12 +246,12 @@ module Sidekiq
|
|
234
246
|
page = 0
|
235
247
|
page_size = 50
|
236
248
|
|
237
|
-
|
249
|
+
loop do
|
238
250
|
range_start = page * page_size - deleted_size
|
239
|
-
range_end
|
240
|
-
entries = Sidekiq.redis
|
251
|
+
range_end = range_start + page_size - 1
|
252
|
+
entries = Sidekiq.redis { |conn|
|
241
253
|
conn.lrange @rname, range_start, range_end
|
242
|
-
|
254
|
+
}
|
243
255
|
break if entries.empty?
|
244
256
|
page += 1
|
245
257
|
entries.each do |entry|
|
@@ -261,8 +273,8 @@ module Sidekiq
|
|
261
273
|
def clear
|
262
274
|
Sidekiq.redis do |conn|
|
263
275
|
conn.multi do
|
264
|
-
conn.
|
265
|
-
conn.srem("queues"
|
276
|
+
conn.unlink(@rname)
|
277
|
+
conn.srem("queues", name)
|
266
278
|
end
|
267
279
|
end
|
268
280
|
end
|
@@ -280,14 +292,26 @@ module Sidekiq
|
|
280
292
|
attr_reader :item
|
281
293
|
attr_reader :value
|
282
294
|
|
283
|
-
def initialize(item, queue_name=nil)
|
295
|
+
def initialize(item, queue_name = nil)
|
296
|
+
@args = nil
|
284
297
|
@value = item
|
285
|
-
@item = item.is_a?(Hash) ? item :
|
286
|
-
@queue = queue_name || @item[
|
298
|
+
@item = item.is_a?(Hash) ? item : parse(item)
|
299
|
+
@queue = queue_name || @item["queue"]
|
300
|
+
end
|
301
|
+
|
302
|
+
def parse(item)
|
303
|
+
Sidekiq.load_json(item)
|
304
|
+
rescue JSON::ParserError
|
305
|
+
# If the job payload in Redis is invalid JSON, we'll load
|
306
|
+
# the item as an empty hash and store the invalid JSON as
|
307
|
+
# the job 'args' for display in the Web UI.
|
308
|
+
@invalid = true
|
309
|
+
@args = [item]
|
310
|
+
{}
|
287
311
|
end
|
288
312
|
|
289
313
|
def klass
|
290
|
-
|
314
|
+
self["class"]
|
291
315
|
end
|
292
316
|
|
293
317
|
def display_class
|
@@ -298,85 +322,123 @@ module Sidekiq
|
|
298
322
|
"#{target}.#{method}"
|
299
323
|
end
|
300
324
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
301
|
-
job_class = @item[
|
302
|
-
if
|
325
|
+
job_class = @item["wrapped"] || args[0]
|
326
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
303
327
|
# MailerClass#mailer_method
|
304
|
-
args[0][
|
328
|
+
args[0]["arguments"][0..1].join("#")
|
305
329
|
else
|
306
|
-
|
330
|
+
job_class
|
307
331
|
end
|
308
332
|
else
|
309
333
|
klass
|
310
|
-
|
334
|
+
end
|
311
335
|
end
|
312
336
|
|
313
337
|
def display_args
|
314
338
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
315
|
-
@
|
339
|
+
@display_args ||= case klass
|
316
340
|
when /\ASidekiq::Extensions::Delayed/
|
317
341
|
safe_load(args[0], args) do |_, _, arg|
|
318
342
|
arg
|
319
343
|
end
|
320
344
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
321
|
-
job_args =
|
322
|
-
if
|
323
|
-
|
324
|
-
|
345
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
346
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
347
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
348
|
+
job_args.drop(3)
|
349
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
350
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
351
|
+
job_args.drop(3).first["args"]
|
325
352
|
else
|
326
|
-
|
353
|
+
job_args
|
327
354
|
end
|
328
355
|
else
|
356
|
+
if self["encrypt"]
|
357
|
+
# no point in showing 150+ bytes of random garbage
|
358
|
+
args[-1] = "[encrypted data]"
|
359
|
+
end
|
329
360
|
args
|
330
|
-
|
361
|
+
end
|
331
362
|
end
|
332
363
|
|
333
364
|
def args
|
334
|
-
@item[
|
365
|
+
@args || @item["args"]
|
335
366
|
end
|
336
367
|
|
337
368
|
def jid
|
338
|
-
|
369
|
+
self["jid"]
|
339
370
|
end
|
340
371
|
|
341
372
|
def enqueued_at
|
342
|
-
|
373
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
343
374
|
end
|
344
375
|
|
345
376
|
def created_at
|
346
|
-
Time.at(
|
377
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
347
378
|
end
|
348
379
|
|
349
|
-
def
|
350
|
-
|
380
|
+
def tags
|
381
|
+
self["tags"] || []
|
351
382
|
end
|
352
383
|
|
384
|
+
def error_backtrace
|
385
|
+
# Cache nil values
|
386
|
+
if defined?(@error_backtrace)
|
387
|
+
@error_backtrace
|
388
|
+
else
|
389
|
+
value = self["error_backtrace"]
|
390
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
394
|
+
attr_reader :queue
|
395
|
+
|
353
396
|
def latency
|
354
|
-
Time.now.to_f
|
397
|
+
now = Time.now.to_f
|
398
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
355
399
|
end
|
356
400
|
|
357
401
|
##
|
358
402
|
# Remove this job from the queue.
|
359
403
|
def delete
|
360
|
-
count = Sidekiq.redis
|
404
|
+
count = Sidekiq.redis { |conn|
|
361
405
|
conn.lrem("queue:#{@queue}", 1, @value)
|
362
|
-
|
406
|
+
}
|
363
407
|
count != 0
|
364
408
|
end
|
365
409
|
|
366
410
|
def [](name)
|
367
|
-
|
411
|
+
# nil will happen if the JSON fails to parse.
|
412
|
+
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
413
|
+
# make a best effort to minimize the damage.
|
414
|
+
@item ? @item[name] : nil
|
368
415
|
end
|
369
416
|
|
370
417
|
private
|
371
418
|
|
372
419
|
def safe_load(content, default)
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
420
|
+
yield(*YAML.load(content))
|
421
|
+
rescue => ex
|
422
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
423
|
+
# memory yet so the YAML can't be loaded.
|
424
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
425
|
+
default
|
426
|
+
end
|
427
|
+
|
428
|
+
def uncompress_backtrace(backtrace)
|
429
|
+
if backtrace.is_a?(Array)
|
430
|
+
# Handle old jobs with raw Array backtrace format
|
431
|
+
backtrace
|
432
|
+
else
|
433
|
+
decoded = Base64.decode64(backtrace)
|
434
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
435
|
+
begin
|
436
|
+
Sidekiq.load_json(uncompressed)
|
437
|
+
rescue
|
438
|
+
# Handle old jobs with marshalled backtrace format
|
439
|
+
# TODO Remove in 7.x
|
440
|
+
Marshal.load(uncompressed)
|
441
|
+
end
|
380
442
|
end
|
381
443
|
end
|
382
444
|
end
|
@@ -404,8 +466,9 @@ module Sidekiq
|
|
404
466
|
end
|
405
467
|
|
406
468
|
def reschedule(at)
|
407
|
-
|
408
|
-
|
469
|
+
Sidekiq.redis do |conn|
|
470
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
471
|
+
end
|
409
472
|
end
|
410
473
|
|
411
474
|
def add_to_queue
|
@@ -416,10 +479,9 @@ module Sidekiq
|
|
416
479
|
end
|
417
480
|
|
418
481
|
def retry
|
419
|
-
raise "Retry not available on jobs which have not failed" unless item["failed_at"]
|
420
482
|
remove_job do |message|
|
421
483
|
msg = Sidekiq.load_json(message)
|
422
|
-
msg[
|
484
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
423
485
|
Sidekiq::Client.push(msg)
|
424
486
|
end
|
425
487
|
end
|
@@ -427,56 +489,50 @@ module Sidekiq
|
|
427
489
|
##
|
428
490
|
# Place job in the dead set
|
429
491
|
def kill
|
430
|
-
raise 'Kill not available on jobs which have not failed' unless item['failed_at']
|
431
492
|
remove_job do |message|
|
432
|
-
|
433
|
-
now = Time.now.to_f
|
434
|
-
Sidekiq.redis do |conn|
|
435
|
-
conn.multi do
|
436
|
-
conn.zadd('dead', now, message)
|
437
|
-
conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
|
438
|
-
conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
|
439
|
-
end
|
440
|
-
end
|
493
|
+
DeadSet.new.kill(message)
|
441
494
|
end
|
442
495
|
end
|
443
496
|
|
497
|
+
def error?
|
498
|
+
!!item["error_class"]
|
499
|
+
end
|
500
|
+
|
444
501
|
private
|
445
502
|
|
446
503
|
def remove_job
|
447
504
|
Sidekiq.redis do |conn|
|
448
|
-
results = conn.multi
|
505
|
+
results = conn.multi {
|
449
506
|
conn.zrangebyscore(parent.name, score, score)
|
450
507
|
conn.zremrangebyscore(parent.name, score, score)
|
451
|
-
|
508
|
+
}.first
|
452
509
|
|
453
510
|
if results.size == 1
|
454
511
|
yield results.first
|
455
512
|
else
|
456
513
|
# multiple jobs with the same score
|
457
514
|
# find the one with the right JID and push it
|
458
|
-
|
515
|
+
matched, nonmatched = results.partition { |message|
|
459
516
|
if message.index(jid)
|
460
517
|
msg = Sidekiq.load_json(message)
|
461
|
-
msg[
|
518
|
+
msg["jid"] == jid
|
462
519
|
else
|
463
520
|
false
|
464
521
|
end
|
465
|
-
|
522
|
+
}
|
466
523
|
|
467
|
-
msg =
|
524
|
+
msg = matched.first
|
468
525
|
yield msg if msg
|
469
526
|
|
470
527
|
# push the rest back onto the sorted set
|
471
528
|
conn.multi do
|
472
|
-
|
529
|
+
nonmatched.each do |message|
|
473
530
|
conn.zadd(parent.name, score.to_f.to_s, message)
|
474
531
|
end
|
475
532
|
end
|
476
533
|
end
|
477
534
|
end
|
478
535
|
end
|
479
|
-
|
480
536
|
end
|
481
537
|
|
482
538
|
class SortedSet
|
@@ -493,16 +549,26 @@ module Sidekiq
|
|
493
549
|
Sidekiq.redis { |c| c.zcard(name) }
|
494
550
|
end
|
495
551
|
|
552
|
+
def scan(match, count = 100)
|
553
|
+
return to_enum(:scan, match, count) unless block_given?
|
554
|
+
|
555
|
+
match = "*#{match}*" unless match.include?("*")
|
556
|
+
Sidekiq.redis do |conn|
|
557
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
558
|
+
yield SortedEntry.new(self, score, entry)
|
559
|
+
end
|
560
|
+
end
|
561
|
+
end
|
562
|
+
|
496
563
|
def clear
|
497
564
|
Sidekiq.redis do |conn|
|
498
|
-
conn.
|
565
|
+
conn.unlink(name)
|
499
566
|
end
|
500
567
|
end
|
501
568
|
alias_method :💣, :clear
|
502
569
|
end
|
503
570
|
|
504
571
|
class JobSet < SortedSet
|
505
|
-
|
506
572
|
def schedule(timestamp, message)
|
507
573
|
Sidekiq.redis do |conn|
|
508
574
|
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
|
@@ -515,44 +581,55 @@ module Sidekiq
|
|
515
581
|
page = -1
|
516
582
|
page_size = 50
|
517
583
|
|
518
|
-
|
584
|
+
loop do
|
519
585
|
range_start = page * page_size + offset_size
|
520
|
-
range_end
|
521
|
-
elements = Sidekiq.redis
|
586
|
+
range_end = range_start + page_size - 1
|
587
|
+
elements = Sidekiq.redis { |conn|
|
522
588
|
conn.zrange name, range_start, range_end, with_scores: true
|
523
|
-
|
589
|
+
}
|
524
590
|
break if elements.empty?
|
525
591
|
page -= 1
|
526
|
-
elements.
|
592
|
+
elements.reverse_each do |element, score|
|
527
593
|
yield SortedEntry.new(self, score, element)
|
528
594
|
end
|
529
595
|
offset_size = initial_size - @_size
|
530
596
|
end
|
531
597
|
end
|
532
598
|
|
599
|
+
##
|
600
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
601
|
+
# optional second argument.
|
533
602
|
def fetch(score, jid = nil)
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
elements.inject([]) do |result, element|
|
539
|
-
entry = SortedEntry.new(self, score, element)
|
540
|
-
if jid
|
541
|
-
result << entry if entry.jid == jid
|
603
|
+
begin_score, end_score =
|
604
|
+
if score.is_a?(Range)
|
605
|
+
[score.first, score.last]
|
542
606
|
else
|
543
|
-
|
607
|
+
[score, score]
|
544
608
|
end
|
545
|
-
|
609
|
+
|
610
|
+
elements = Sidekiq.redis { |conn|
|
611
|
+
conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
|
612
|
+
}
|
613
|
+
|
614
|
+
elements.each_with_object([]) do |element, result|
|
615
|
+
data, job_score = element
|
616
|
+
entry = SortedEntry.new(self, job_score, data)
|
617
|
+
result << entry if jid.nil? || entry.jid == jid
|
546
618
|
end
|
547
619
|
end
|
548
620
|
|
549
621
|
##
|
550
622
|
# Find the job with the given JID within this sorted set.
|
551
|
-
#
|
552
|
-
# This is a slow, inefficient operation. Do not use under
|
553
|
-
# normal conditions. Sidekiq Pro contains a faster version.
|
623
|
+
# This is a slower O(n) operation. Do not use for app logic.
|
554
624
|
def find_job(jid)
|
555
|
-
|
625
|
+
Sidekiq.redis do |conn|
|
626
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
627
|
+
job = JSON.parse(entry)
|
628
|
+
matched = job["jid"] == jid
|
629
|
+
return SortedEntry.new(self, score, entry) if matched
|
630
|
+
end
|
631
|
+
end
|
632
|
+
nil
|
556
633
|
end
|
557
634
|
|
558
635
|
def delete_by_value(name, value)
|
@@ -567,13 +644,14 @@ module Sidekiq
|
|
567
644
|
Sidekiq.redis do |conn|
|
568
645
|
elements = conn.zrangebyscore(name, score, score)
|
569
646
|
elements.each do |element|
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
647
|
+
if element.index(jid)
|
648
|
+
message = Sidekiq.load_json(element)
|
649
|
+
if message["jid"] == jid
|
650
|
+
ret = conn.zrem(name, element)
|
651
|
+
@_size -= 1 if ret
|
652
|
+
break ret
|
653
|
+
end
|
575
654
|
end
|
576
|
-
false
|
577
655
|
end
|
578
656
|
end
|
579
657
|
end
|
@@ -585,17 +663,17 @@ module Sidekiq
|
|
585
663
|
# Allows enumeration of scheduled jobs within Sidekiq.
|
586
664
|
# Based on this, you can search/filter for jobs. Here's an
|
587
665
|
# example where I'm selecting all jobs of a certain type
|
588
|
-
# and deleting them from the
|
666
|
+
# and deleting them from the schedule queue.
|
589
667
|
#
|
590
668
|
# r = Sidekiq::ScheduledSet.new
|
591
|
-
# r.select do |
|
592
|
-
#
|
593
|
-
#
|
594
|
-
#
|
669
|
+
# r.select do |scheduled|
|
670
|
+
# scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
|
671
|
+
# scheduled.args[0] == 'User' &&
|
672
|
+
# scheduled.args[1] == 'setup_new_subscriber'
|
595
673
|
# end.map(&:delete)
|
596
674
|
class ScheduledSet < JobSet
|
597
675
|
def initialize
|
598
|
-
super
|
676
|
+
super "schedule"
|
599
677
|
end
|
600
678
|
end
|
601
679
|
|
@@ -613,13 +691,15 @@ module Sidekiq
|
|
613
691
|
# end.map(&:delete)
|
614
692
|
class RetrySet < JobSet
|
615
693
|
def initialize
|
616
|
-
super
|
694
|
+
super "retry"
|
617
695
|
end
|
618
696
|
|
619
697
|
def retry_all
|
620
|
-
while size > 0
|
621
|
-
|
622
|
-
|
698
|
+
each(&:retry) while size > 0
|
699
|
+
end
|
700
|
+
|
701
|
+
def kill_all
|
702
|
+
each(&:kill) while size > 0
|
623
703
|
end
|
624
704
|
end
|
625
705
|
|
@@ -628,13 +708,32 @@ module Sidekiq
|
|
628
708
|
#
|
629
709
|
class DeadSet < JobSet
|
630
710
|
def initialize
|
631
|
-
super
|
711
|
+
super "dead"
|
632
712
|
end
|
633
713
|
|
634
|
-
def
|
635
|
-
|
636
|
-
|
714
|
+
def kill(message, opts = {})
|
715
|
+
now = Time.now.to_f
|
716
|
+
Sidekiq.redis do |conn|
|
717
|
+
conn.multi do
|
718
|
+
conn.zadd(name, now.to_s, message)
|
719
|
+
conn.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
720
|
+
conn.zremrangebyrank(name, 0, - self.class.max_jobs)
|
721
|
+
end
|
637
722
|
end
|
723
|
+
|
724
|
+
if opts[:notify_failure] != false
|
725
|
+
job = Sidekiq.load_json(message)
|
726
|
+
r = RuntimeError.new("Job killed by API")
|
727
|
+
r.set_backtrace(caller)
|
728
|
+
Sidekiq.death_handlers.each do |handle|
|
729
|
+
handle.call(job, r)
|
730
|
+
end
|
731
|
+
end
|
732
|
+
true
|
733
|
+
end
|
734
|
+
|
735
|
+
def retry_all
|
736
|
+
each(&:retry) while size > 0
|
638
737
|
end
|
639
738
|
|
640
739
|
def self.max_jobs
|
@@ -648,7 +747,7 @@ module Sidekiq
|
|
648
747
|
|
649
748
|
##
|
650
749
|
# Enumerates the set of Sidekiq processes which are actively working
|
651
|
-
# right now. Each process
|
750
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
652
751
|
# so this set should be relatively accurate, barring network partitions.
|
653
752
|
#
|
654
753
|
# Yields a Sidekiq::Process.
|
@@ -656,54 +755,60 @@ module Sidekiq
|
|
656
755
|
class ProcessSet
|
657
756
|
include Enumerable
|
658
757
|
|
659
|
-
def initialize(clean_plz=true)
|
660
|
-
|
758
|
+
def initialize(clean_plz = true)
|
759
|
+
cleanup if clean_plz
|
661
760
|
end
|
662
761
|
|
663
762
|
# Cleans up dead processes recorded in Redis.
|
664
763
|
# Returns the number of processes cleaned.
|
665
|
-
def
|
764
|
+
def cleanup
|
666
765
|
count = 0
|
667
766
|
Sidekiq.redis do |conn|
|
668
|
-
procs = conn.
|
669
|
-
heartbeats = conn.pipelined
|
767
|
+
procs = conn.sscan_each("processes").to_a.sort
|
768
|
+
heartbeats = conn.pipelined {
|
670
769
|
procs.each do |key|
|
671
|
-
conn.hget(key,
|
770
|
+
conn.hget(key, "info")
|
672
771
|
end
|
673
|
-
|
772
|
+
}
|
674
773
|
|
675
774
|
# the hash named key has an expiry of 60 seconds.
|
676
775
|
# if it's not found, that means the process has not reported
|
677
776
|
# in to Redis and probably died.
|
678
|
-
to_prune =
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
count = conn.srem('processes', to_prune) unless to_prune.empty?
|
777
|
+
to_prune = procs.select.with_index { |proc, i|
|
778
|
+
heartbeats[i].nil?
|
779
|
+
}
|
780
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
683
781
|
end
|
684
782
|
count
|
685
783
|
end
|
686
784
|
|
687
785
|
def each
|
688
|
-
|
786
|
+
result = Sidekiq.redis { |conn|
|
787
|
+
procs = conn.sscan_each("processes").to_a.sort
|
689
788
|
|
690
|
-
Sidekiq.redis do |conn|
|
691
789
|
# We're making a tradeoff here between consuming more memory instead of
|
692
790
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
693
791
|
# you'll be happier this way
|
694
|
-
|
792
|
+
conn.pipelined do
|
695
793
|
procs.each do |key|
|
696
|
-
conn.hmget(key,
|
794
|
+
conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
697
795
|
end
|
698
796
|
end
|
797
|
+
}
|
699
798
|
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
799
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
800
|
+
# If a process is stopped between when we query Redis for `procs` and
|
801
|
+
# when we query for `result`, we will have an item in `result` that is
|
802
|
+
# composed of `nil` values.
|
803
|
+
next if info.nil?
|
705
804
|
|
706
|
-
|
805
|
+
hash = Sidekiq.load_json(info)
|
806
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
807
|
+
"beat" => at_s.to_f,
|
808
|
+
"quiet" => quiet,
|
809
|
+
"rss" => rss.to_i,
|
810
|
+
"rtt_us" => rtt.to_i))
|
811
|
+
end
|
707
812
|
end
|
708
813
|
|
709
814
|
# This method is not guaranteed accurate since it does not prune the set
|
@@ -711,7 +816,30 @@ module Sidekiq
|
|
711
816
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
712
817
|
# 60 seconds.
|
713
818
|
def size
|
714
|
-
Sidekiq.redis { |conn| conn.scard(
|
819
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
820
|
+
end
|
821
|
+
|
822
|
+
# Total number of threads available to execute jobs.
|
823
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
824
|
+
# less than or equal to your licensed concurrency.
|
825
|
+
def total_concurrency
|
826
|
+
sum { |x| x["concurrency"] }
|
827
|
+
end
|
828
|
+
|
829
|
+
def total_rss
|
830
|
+
sum { |x| x["rss"] || 0 }
|
831
|
+
end
|
832
|
+
|
833
|
+
# Returns the identity of the current cluster leader or "" if no leader.
|
834
|
+
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
835
|
+
# or Sidekiq Pro.
|
836
|
+
def leader
|
837
|
+
@leader ||= begin
|
838
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
839
|
+
# need a non-falsy value so we can memoize
|
840
|
+
x ||= ""
|
841
|
+
x
|
842
|
+
end
|
715
843
|
end
|
716
844
|
end
|
717
845
|
|
@@ -736,31 +864,35 @@ module Sidekiq
|
|
736
864
|
end
|
737
865
|
|
738
866
|
def tag
|
739
|
-
self[
|
867
|
+
self["tag"]
|
740
868
|
end
|
741
869
|
|
742
870
|
def labels
|
743
|
-
Array(self[
|
871
|
+
Array(self["labels"])
|
744
872
|
end
|
745
873
|
|
746
874
|
def [](key)
|
747
875
|
@attribs[key]
|
748
876
|
end
|
749
877
|
|
878
|
+
def identity
|
879
|
+
self["identity"]
|
880
|
+
end
|
881
|
+
|
750
882
|
def quiet!
|
751
|
-
signal(
|
883
|
+
signal("TSTP")
|
752
884
|
end
|
753
885
|
|
754
886
|
def stop!
|
755
|
-
signal(
|
887
|
+
signal("TERM")
|
756
888
|
end
|
757
889
|
|
758
890
|
def dump_threads
|
759
|
-
signal(
|
891
|
+
signal("TTIN")
|
760
892
|
end
|
761
893
|
|
762
894
|
def stopping?
|
763
|
-
self[
|
895
|
+
self["quiet"] == "true"
|
764
896
|
end
|
765
897
|
|
766
898
|
private
|
@@ -774,10 +906,6 @@ module Sidekiq
|
|
774
906
|
end
|
775
907
|
end
|
776
908
|
end
|
777
|
-
|
778
|
-
def identity
|
779
|
-
self['identity']
|
780
|
-
end
|
781
909
|
end
|
782
910
|
|
783
911
|
##
|
@@ -803,20 +931,27 @@ module Sidekiq
|
|
803
931
|
class Workers
|
804
932
|
include Enumerable
|
805
933
|
|
806
|
-
def each
|
934
|
+
def each(&block)
|
935
|
+
results = []
|
807
936
|
Sidekiq.redis do |conn|
|
808
|
-
procs = conn.
|
937
|
+
procs = conn.sscan_each("processes").to_a
|
809
938
|
procs.sort.each do |key|
|
810
|
-
valid, workers = conn.pipelined
|
811
|
-
conn.exists(key)
|
939
|
+
valid, workers = conn.pipelined {
|
940
|
+
conn.exists?(key)
|
812
941
|
conn.hgetall("#{key}:workers")
|
813
|
-
|
942
|
+
}
|
814
943
|
next unless valid
|
815
944
|
workers.each_pair do |tid, json|
|
816
|
-
|
945
|
+
hsh = Sidekiq.load_json(json)
|
946
|
+
p = hsh["payload"]
|
947
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
948
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
949
|
+
results << [key, tid, hsh]
|
817
950
|
end
|
818
951
|
end
|
819
952
|
end
|
953
|
+
|
954
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
820
955
|
end
|
821
956
|
|
822
957
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -827,18 +962,17 @@ module Sidekiq
|
|
827
962
|
# which can easily get out of sync with crashy processes.
|
828
963
|
def size
|
829
964
|
Sidekiq.redis do |conn|
|
830
|
-
procs = conn.
|
965
|
+
procs = conn.sscan_each("processes").to_a
|
831
966
|
if procs.empty?
|
832
967
|
0
|
833
968
|
else
|
834
|
-
conn.pipelined
|
969
|
+
conn.pipelined {
|
835
970
|
procs.each do |key|
|
836
|
-
conn.hget(key,
|
971
|
+
conn.hget(key, "busy")
|
837
972
|
end
|
838
|
-
|
973
|
+
}.sum(&:to_i)
|
839
974
|
end
|
840
975
|
end
|
841
976
|
end
|
842
977
|
end
|
843
|
-
|
844
978
|
end
|