sidekiq 6.0.7 → 6.5.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +209 -2
- data/LICENSE +3 -3
- data/README.md +11 -10
- data/bin/sidekiq +8 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +180 -123
- data/lib/sidekiq/cli.rb +80 -45
- data/lib/sidekiq/client.rb +52 -71
- data/lib/sidekiq/{util.rb → component.rb} +11 -14
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +41 -30
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +36 -36
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +123 -63
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +28 -17
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +19 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +41 -41
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +87 -53
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +39 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +40 -34
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +136 -16
- data/lib/sidekiq.rb +107 -30
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +113 -65
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +42 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +51 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +29 -51
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/api.rb
CHANGED
@@ -8,7 +8,7 @@ require "base64"
|
|
8
8
|
module Sidekiq
|
9
9
|
class Stats
|
10
10
|
def initialize
|
11
|
-
|
11
|
+
fetch_stats_fast!
|
12
12
|
end
|
13
13
|
|
14
14
|
def processed
|
@@ -51,50 +51,33 @@ module Sidekiq
|
|
51
51
|
Sidekiq::Stats::Queues.new.lengths
|
52
52
|
end
|
53
53
|
|
54
|
-
|
54
|
+
# O(1) redis calls
|
55
|
+
def fetch_stats_fast!
|
55
56
|
pipe1_res = Sidekiq.redis { |conn|
|
56
|
-
conn.pipelined do
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
end
|
65
|
-
}
|
66
|
-
|
67
|
-
processes = Sidekiq.redis { |conn|
|
68
|
-
conn.sscan_each("processes").to_a
|
69
|
-
}
|
70
|
-
|
71
|
-
queues = Sidekiq.redis { |conn|
|
72
|
-
conn.sscan_each("queues").to_a
|
73
|
-
}
|
74
|
-
|
75
|
-
pipe2_res = Sidekiq.redis { |conn|
|
76
|
-
conn.pipelined do
|
77
|
-
processes.each { |key| conn.hget(key, "busy") }
|
78
|
-
queues.each { |queue| conn.llen("queue:#{queue}") }
|
57
|
+
conn.pipelined do |pipeline|
|
58
|
+
pipeline.get("stat:processed")
|
59
|
+
pipeline.get("stat:failed")
|
60
|
+
pipeline.zcard("schedule")
|
61
|
+
pipeline.zcard("retry")
|
62
|
+
pipeline.zcard("dead")
|
63
|
+
pipeline.scard("processes")
|
64
|
+
pipeline.lrange("queue:default", -1, -1)
|
79
65
|
end
|
80
66
|
}
|
81
67
|
|
82
|
-
s = processes.size
|
83
|
-
workers_size = pipe2_res[0...s].sum(&:to_i)
|
84
|
-
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
85
|
-
|
86
68
|
default_queue_latency = if (entry = pipe1_res[6].first)
|
87
69
|
job = begin
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
70
|
+
Sidekiq.load_json(entry)
|
71
|
+
rescue
|
72
|
+
{}
|
73
|
+
end
|
92
74
|
now = Time.now.to_f
|
93
75
|
thence = job["enqueued_at"] || now
|
94
76
|
now - thence
|
95
77
|
else
|
96
78
|
0
|
97
79
|
end
|
80
|
+
|
98
81
|
@stats = {
|
99
82
|
processed: pipe1_res[0].to_i,
|
100
83
|
failed: pipe1_res[1].to_i,
|
@@ -103,10 +86,39 @@ module Sidekiq
|
|
103
86
|
dead_size: pipe1_res[4],
|
104
87
|
processes_size: pipe1_res[5],
|
105
88
|
|
106
|
-
default_queue_latency: default_queue_latency
|
107
|
-
|
108
|
-
|
89
|
+
default_queue_latency: default_queue_latency
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
# O(number of processes + number of queues) redis calls
|
94
|
+
def fetch_stats_slow!
|
95
|
+
processes = Sidekiq.redis { |conn|
|
96
|
+
conn.sscan_each("processes").to_a
|
97
|
+
}
|
98
|
+
|
99
|
+
queues = Sidekiq.redis { |conn|
|
100
|
+
conn.sscan_each("queues").to_a
|
101
|
+
}
|
102
|
+
|
103
|
+
pipe2_res = Sidekiq.redis { |conn|
|
104
|
+
conn.pipelined do |pipeline|
|
105
|
+
processes.each { |key| pipeline.hget(key, "busy") }
|
106
|
+
queues.each { |queue| pipeline.llen("queue:#{queue}") }
|
107
|
+
end
|
109
108
|
}
|
109
|
+
|
110
|
+
s = processes.size
|
111
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
112
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
113
|
+
|
114
|
+
@stats[:workers_size] = workers_size
|
115
|
+
@stats[:enqueued] = enqueued
|
116
|
+
@stats
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_stats!
|
120
|
+
fetch_stats_fast!
|
121
|
+
fetch_stats_slow!
|
110
122
|
end
|
111
123
|
|
112
124
|
def reset(*stats)
|
@@ -126,7 +138,8 @@ module Sidekiq
|
|
126
138
|
private
|
127
139
|
|
128
140
|
def stat(s)
|
129
|
-
@stats[s]
|
141
|
+
fetch_stats_slow! if @stats[s].nil?
|
142
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
130
143
|
end
|
131
144
|
|
132
145
|
class Queues
|
@@ -134,20 +147,22 @@ module Sidekiq
|
|
134
147
|
Sidekiq.redis do |conn|
|
135
148
|
queues = conn.sscan_each("queues").to_a
|
136
149
|
|
137
|
-
lengths = conn.pipelined {
|
150
|
+
lengths = conn.pipelined { |pipeline|
|
138
151
|
queues.each do |queue|
|
139
|
-
|
152
|
+
pipeline.llen("queue:#{queue}")
|
140
153
|
end
|
141
154
|
}
|
142
155
|
|
143
156
|
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
144
|
-
|
157
|
+
array_of_arrays.to_h
|
145
158
|
end
|
146
159
|
end
|
147
160
|
end
|
148
161
|
|
149
162
|
class History
|
150
163
|
def initialize(days_previous, start_date = nil)
|
164
|
+
# we only store five years of data in Redis
|
165
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
151
166
|
@days_previous = days_previous
|
152
167
|
@start_date = start_date || Time.now.utc.to_date
|
153
168
|
end
|
@@ -176,7 +191,7 @@ module Sidekiq
|
|
176
191
|
stat_hash[dates[idx]] = value ? value.to_i : 0
|
177
192
|
end
|
178
193
|
end
|
179
|
-
rescue
|
194
|
+
rescue RedisConnection.adapter::CommandError
|
180
195
|
# mget will trigger a CROSSSLOT error when run against a Cluster
|
181
196
|
# TODO Someone want to add Cluster support?
|
182
197
|
end
|
@@ -255,7 +270,7 @@ module Sidekiq
|
|
255
270
|
break if entries.empty?
|
256
271
|
page += 1
|
257
272
|
entries.each do |entry|
|
258
|
-
yield
|
273
|
+
yield JobRecord.new(entry, @name)
|
259
274
|
end
|
260
275
|
deleted_size = initial_size - size
|
261
276
|
end
|
@@ -265,20 +280,24 @@ module Sidekiq
|
|
265
280
|
# Find the job with the given JID within this queue.
|
266
281
|
#
|
267
282
|
# This is a slow, inefficient operation. Do not use under
|
268
|
-
# normal conditions.
|
283
|
+
# normal conditions.
|
269
284
|
def find_job(jid)
|
270
285
|
detect { |j| j.jid == jid }
|
271
286
|
end
|
272
287
|
|
273
288
|
def clear
|
274
289
|
Sidekiq.redis do |conn|
|
275
|
-
conn.multi do
|
276
|
-
|
277
|
-
|
290
|
+
conn.multi do |transaction|
|
291
|
+
transaction.unlink(@rname)
|
292
|
+
transaction.srem("queues", name)
|
278
293
|
end
|
279
294
|
end
|
280
295
|
end
|
281
296
|
alias_method :💣, :clear
|
297
|
+
|
298
|
+
def as_json(options = nil) # :nodoc:
|
299
|
+
{name: name} # 5336
|
300
|
+
end
|
282
301
|
end
|
283
302
|
|
284
303
|
##
|
@@ -286,9 +305,9 @@ module Sidekiq
|
|
286
305
|
# sorted set.
|
287
306
|
#
|
288
307
|
# The job should be considered immutable but may be
|
289
|
-
# removed from the queue via
|
308
|
+
# removed from the queue via JobRecord#delete.
|
290
309
|
#
|
291
|
-
class
|
310
|
+
class JobRecord
|
292
311
|
attr_reader :item
|
293
312
|
attr_reader :value
|
294
313
|
|
@@ -316,48 +335,54 @@ module Sidekiq
|
|
316
335
|
|
317
336
|
def display_class
|
318
337
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
319
|
-
@klass ||=
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
338
|
+
@klass ||= self["display_class"] || begin
|
339
|
+
case klass
|
340
|
+
when /\ASidekiq::Extensions::Delayed/
|
341
|
+
safe_load(args[0], klass) do |target, method, _|
|
342
|
+
"#{target}.#{method}"
|
343
|
+
end
|
344
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
345
|
+
job_class = @item["wrapped"] || args[0]
|
346
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
347
|
+
# MailerClass#mailer_method
|
348
|
+
args[0]["arguments"][0..1].join("#")
|
349
|
+
else
|
350
|
+
job_class
|
351
|
+
end
|
352
|
+
else
|
353
|
+
klass
|
354
|
+
end
|
334
355
|
end
|
335
356
|
end
|
336
357
|
|
337
358
|
def display_args
|
338
359
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
339
360
|
@display_args ||= case klass
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
+
when /\ASidekiq::Extensions::Delayed/
|
362
|
+
safe_load(args[0], args) do |_, _, arg, kwarg|
|
363
|
+
if !kwarg || kwarg.empty?
|
364
|
+
arg
|
365
|
+
else
|
366
|
+
[arg, kwarg]
|
367
|
+
end
|
368
|
+
end
|
369
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
370
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
371
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
372
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
373
|
+
job_args.drop(3)
|
374
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
375
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
376
|
+
job_args.drop(3).first["args"]
|
377
|
+
else
|
378
|
+
job_args
|
379
|
+
end
|
380
|
+
else
|
381
|
+
if self["encrypt"]
|
382
|
+
# no point in showing 150+ bytes of random garbage
|
383
|
+
args[-1] = "[encrypted data]"
|
384
|
+
end
|
385
|
+
args
|
361
386
|
end
|
362
387
|
end
|
363
388
|
|
@@ -421,7 +446,8 @@ module Sidekiq
|
|
421
446
|
rescue => ex
|
422
447
|
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
423
448
|
# memory yet so the YAML can't be loaded.
|
424
|
-
|
449
|
+
# TODO is this still necessary? Zeitwerk reloader should handle?
|
450
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
|
425
451
|
default
|
426
452
|
end
|
427
453
|
|
@@ -443,13 +469,13 @@ module Sidekiq
|
|
443
469
|
end
|
444
470
|
end
|
445
471
|
|
446
|
-
class SortedEntry <
|
472
|
+
class SortedEntry < JobRecord
|
447
473
|
attr_reader :score
|
448
474
|
attr_reader :parent
|
449
475
|
|
450
476
|
def initialize(parent, score, item)
|
451
477
|
super(item)
|
452
|
-
@score = score
|
478
|
+
@score = Float(score)
|
453
479
|
@parent = parent
|
454
480
|
end
|
455
481
|
|
@@ -502,9 +528,9 @@ module Sidekiq
|
|
502
528
|
|
503
529
|
def remove_job
|
504
530
|
Sidekiq.redis do |conn|
|
505
|
-
results = conn.multi {
|
506
|
-
|
507
|
-
|
531
|
+
results = conn.multi { |transaction|
|
532
|
+
transaction.zrangebyscore(parent.name, score, score)
|
533
|
+
transaction.zremrangebyscore(parent.name, score, score)
|
508
534
|
}.first
|
509
535
|
|
510
536
|
if results.size == 1
|
@@ -525,9 +551,9 @@ module Sidekiq
|
|
525
551
|
yield msg if msg
|
526
552
|
|
527
553
|
# push the rest back onto the sorted set
|
528
|
-
conn.multi do
|
554
|
+
conn.multi do |transaction|
|
529
555
|
nonmatched.each do |message|
|
530
|
-
|
556
|
+
transaction.zadd(parent.name, score.to_f.to_s, message)
|
531
557
|
end
|
532
558
|
end
|
533
559
|
end
|
@@ -566,6 +592,10 @@ module Sidekiq
|
|
566
592
|
end
|
567
593
|
end
|
568
594
|
alias_method :💣, :clear
|
595
|
+
|
596
|
+
def as_json(options = nil) # :nodoc:
|
597
|
+
{name: name} # 5336
|
598
|
+
end
|
569
599
|
end
|
570
600
|
|
571
601
|
class JobSet < SortedSet
|
@@ -585,7 +615,7 @@ module Sidekiq
|
|
585
615
|
range_start = page * page_size + offset_size
|
586
616
|
range_end = range_start + page_size - 1
|
587
617
|
elements = Sidekiq.redis { |conn|
|
588
|
-
conn.zrange name, range_start, range_end,
|
618
|
+
conn.zrange name, range_start, range_end, withscores: true
|
589
619
|
}
|
590
620
|
break if elements.empty?
|
591
621
|
page -= 1
|
@@ -608,7 +638,7 @@ module Sidekiq
|
|
608
638
|
end
|
609
639
|
|
610
640
|
elements = Sidekiq.redis { |conn|
|
611
|
-
conn.zrangebyscore(name, begin_score, end_score,
|
641
|
+
conn.zrangebyscore(name, begin_score, end_score, withscores: true)
|
612
642
|
}
|
613
643
|
|
614
644
|
elements.each_with_object([]) do |element, result|
|
@@ -714,10 +744,10 @@ module Sidekiq
|
|
714
744
|
def kill(message, opts = {})
|
715
745
|
now = Time.now.to_f
|
716
746
|
Sidekiq.redis do |conn|
|
717
|
-
conn.multi do
|
718
|
-
|
719
|
-
|
720
|
-
|
747
|
+
conn.multi do |transaction|
|
748
|
+
transaction.zadd(name, now.to_s, message)
|
749
|
+
transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
750
|
+
transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
|
721
751
|
end
|
722
752
|
end
|
723
753
|
|
@@ -737,11 +767,11 @@ module Sidekiq
|
|
737
767
|
end
|
738
768
|
|
739
769
|
def self.max_jobs
|
740
|
-
Sidekiq
|
770
|
+
Sidekiq[:dead_max_jobs]
|
741
771
|
end
|
742
772
|
|
743
773
|
def self.timeout
|
744
|
-
Sidekiq
|
774
|
+
Sidekiq[:dead_timeout_in_seconds]
|
745
775
|
end
|
746
776
|
end
|
747
777
|
|
@@ -765,9 +795,9 @@ module Sidekiq
|
|
765
795
|
count = 0
|
766
796
|
Sidekiq.redis do |conn|
|
767
797
|
procs = conn.sscan_each("processes").to_a.sort
|
768
|
-
heartbeats = conn.pipelined {
|
798
|
+
heartbeats = conn.pipelined { |pipeline|
|
769
799
|
procs.each do |key|
|
770
|
-
|
800
|
+
pipeline.hget(key, "info")
|
771
801
|
end
|
772
802
|
}
|
773
803
|
|
@@ -789,21 +819,25 @@ module Sidekiq
|
|
789
819
|
# We're making a tradeoff here between consuming more memory instead of
|
790
820
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
791
821
|
# you'll be happier this way
|
792
|
-
conn.pipelined do
|
822
|
+
conn.pipelined do |pipeline|
|
793
823
|
procs.each do |key|
|
794
|
-
|
824
|
+
pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
795
825
|
end
|
796
826
|
end
|
797
827
|
}
|
798
828
|
|
799
|
-
result.each do |info, busy, at_s, quiet|
|
829
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
800
830
|
# If a process is stopped between when we query Redis for `procs` and
|
801
831
|
# when we query for `result`, we will have an item in `result` that is
|
802
832
|
# composed of `nil` values.
|
803
833
|
next if info.nil?
|
804
834
|
|
805
835
|
hash = Sidekiq.load_json(info)
|
806
|
-
yield Process.new(hash.merge("busy" => busy.to_i,
|
836
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
837
|
+
"beat" => at_s.to_f,
|
838
|
+
"quiet" => quiet,
|
839
|
+
"rss" => rss.to_i,
|
840
|
+
"rtt_us" => rtt.to_i))
|
807
841
|
end
|
808
842
|
end
|
809
843
|
|
@@ -815,6 +849,18 @@ module Sidekiq
|
|
815
849
|
Sidekiq.redis { |conn| conn.scard("processes") }
|
816
850
|
end
|
817
851
|
|
852
|
+
# Total number of threads available to execute jobs.
|
853
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
854
|
+
# less than or equal to your licensed concurrency.
|
855
|
+
def total_concurrency
|
856
|
+
sum { |x| x["concurrency"].to_i }
|
857
|
+
end
|
858
|
+
|
859
|
+
def total_rss_in_kb
|
860
|
+
sum { |x| x["rss"].to_i }
|
861
|
+
end
|
862
|
+
alias_method :total_rss, :total_rss_in_kb
|
863
|
+
|
818
864
|
# Returns the identity of the current cluster leader or "" if no leader.
|
819
865
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
820
866
|
# or Sidekiq Pro.
|
@@ -864,6 +910,10 @@ module Sidekiq
|
|
864
910
|
self["identity"]
|
865
911
|
end
|
866
912
|
|
913
|
+
def queues
|
914
|
+
self["queues"]
|
915
|
+
end
|
916
|
+
|
867
917
|
def quiet!
|
868
918
|
signal("TSTP")
|
869
919
|
end
|
@@ -885,17 +935,17 @@ module Sidekiq
|
|
885
935
|
def signal(sig)
|
886
936
|
key = "#{identity}-signals"
|
887
937
|
Sidekiq.redis do |c|
|
888
|
-
c.multi do
|
889
|
-
|
890
|
-
|
938
|
+
c.multi do |transaction|
|
939
|
+
transaction.lpush(key, sig)
|
940
|
+
transaction.expire(key, 60)
|
891
941
|
end
|
892
942
|
end
|
893
943
|
end
|
894
944
|
end
|
895
945
|
|
896
946
|
##
|
897
|
-
#
|
898
|
-
#
|
947
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
948
|
+
# It tracks the process and thread working on each job.
|
899
949
|
#
|
900
950
|
# WARNING WARNING WARNING
|
901
951
|
#
|
@@ -903,26 +953,27 @@ module Sidekiq
|
|
903
953
|
# If you call #size => 5 and then expect #each to be
|
904
954
|
# called 5 times, you're going to have a bad time.
|
905
955
|
#
|
906
|
-
#
|
907
|
-
#
|
908
|
-
#
|
956
|
+
# works = Sidekiq::WorkSet.new
|
957
|
+
# works.size => 2
|
958
|
+
# works.each do |process_id, thread_id, work|
|
909
959
|
# # process_id is a unique identifier per Sidekiq process
|
910
960
|
# # thread_id is a unique identifier per thread
|
911
961
|
# # work is a Hash which looks like:
|
912
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
962
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
913
963
|
# # run_at is an epoch Integer.
|
914
964
|
# end
|
915
965
|
#
|
916
|
-
class
|
966
|
+
class WorkSet
|
917
967
|
include Enumerable
|
918
968
|
|
919
|
-
def each
|
969
|
+
def each(&block)
|
970
|
+
results = []
|
920
971
|
Sidekiq.redis do |conn|
|
921
972
|
procs = conn.sscan_each("processes").to_a
|
922
973
|
procs.sort.each do |key|
|
923
|
-
valid, workers = conn.pipelined {
|
924
|
-
|
925
|
-
|
974
|
+
valid, workers = conn.pipelined { |pipeline|
|
975
|
+
pipeline.exists?(key)
|
976
|
+
pipeline.hgetall("#{key}:work")
|
926
977
|
}
|
927
978
|
next unless valid
|
928
979
|
workers.each_pair do |tid, json|
|
@@ -930,10 +981,12 @@ module Sidekiq
|
|
930
981
|
p = hsh["payload"]
|
931
982
|
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
932
983
|
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
933
|
-
|
984
|
+
results << [key, tid, hsh]
|
934
985
|
end
|
935
986
|
end
|
936
987
|
end
|
988
|
+
|
989
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
937
990
|
end
|
938
991
|
|
939
992
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -948,13 +1001,17 @@ module Sidekiq
|
|
948
1001
|
if procs.empty?
|
949
1002
|
0
|
950
1003
|
else
|
951
|
-
conn.pipelined {
|
1004
|
+
conn.pipelined { |pipeline|
|
952
1005
|
procs.each do |key|
|
953
|
-
|
1006
|
+
pipeline.hget(key, "busy")
|
954
1007
|
end
|
955
1008
|
}.sum(&:to_i)
|
956
1009
|
end
|
957
1010
|
end
|
958
1011
|
end
|
959
1012
|
end
|
1013
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1014
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1015
|
+
# WorkSet better describes the data.
|
1016
|
+
Workers = WorkSet
|
960
1017
|
end
|