sidekiq 6.5.1 → 6.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changes.md +65 -0
- data/bin/sidekiqload +2 -2
- data/lib/sidekiq/api.rb +161 -37
- data/lib/sidekiq/cli.rb +13 -0
- data/lib/sidekiq/client.rb +2 -2
- data/lib/sidekiq/component.rb +2 -1
- data/lib/sidekiq/fetch.rb +2 -2
- data/lib/sidekiq/job_retry.rb +55 -35
- data/lib/sidekiq/launcher.rb +6 -4
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +70 -35
- data/lib/sidekiq/middleware/current_attributes.rb +14 -12
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +9 -1
- data/lib/sidekiq/processor.rb +9 -3
- data/lib/sidekiq/rails.rb +10 -11
- data/lib/sidekiq/redis_connection.rb +0 -2
- data/lib/sidekiq/scheduled.rb +43 -15
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +21 -5
- data/lib/sidekiq/web/helpers.rb +17 -4
- data/lib/sidekiq/web.rb +5 -1
- data/lib/sidekiq/worker.rb +6 -3
- data/lib/sidekiq.rb +9 -1
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +2 -1
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application.css +44 -1
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/ja.yml +7 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_nav.erb +1 -1
- data/web/views/busy.erb +7 -2
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +29 -8
- data/lib/sidekiq/.DS_Store +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 55c6f9a8c0f2810bcbe7744c95204893d73f534666f6091c74dcb5199e7a4a28
|
4
|
+
data.tar.gz: c42690ef0d1876c94eb51fb68319275d11f10169826180db921b34d6334a1a54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 461b559e18cbdf8fe6ba20ab9fa38d08fd3b7b8d14dce7a7f9369d678e92bd25c07734bc14b0167f83589c0f03501897195b3fdf9cfd5de5a60f039bf7436f98
|
7
|
+
data.tar.gz: ce0490b438a22a71c37e5a65193a7728e3297b437730d30a3653807ff1e89476db1f8c62d8de50c0c8cebbbce2ba6fb65f110855e071c62746fe8697a0f636e1
|
data/Changes.md
CHANGED
@@ -2,6 +2,71 @@
|
|
2
2
|
|
3
3
|
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
+
6.5.11
|
6
|
+
----------
|
7
|
+
|
8
|
+
- Fix for Rails 7.1 [#6067]
|
9
|
+
|
10
|
+
6.5.10
|
11
|
+
----------
|
12
|
+
|
13
|
+
- Web UI DoS vector [#6045] CVE-2023-26141
|
14
|
+
- Fix broadcast logger with Rails 7.1 [#6054]
|
15
|
+
|
16
|
+
6.5.9
|
17
|
+
----------
|
18
|
+
|
19
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
20
|
+
|
21
|
+
6.5.8
|
22
|
+
----------
|
23
|
+
|
24
|
+
- Fail if using a bad version of scout_apm [#5616]
|
25
|
+
- Add pagination to Busy page [#5556]
|
26
|
+
- Speed up WorkSet#each [#5559]
|
27
|
+
- Adjust CurrentAttributes to work with the String class name so we aren't referencing
|
28
|
+
the Class within a Rails initializer [#5536]
|
29
|
+
|
30
|
+
6.5.7
|
31
|
+
----------
|
32
|
+
|
33
|
+
- Updates for JA and ZH locales
|
34
|
+
- Further optimizations for scheduled polling [#5513]
|
35
|
+
|
36
|
+
6.5.6
|
37
|
+
----------
|
38
|
+
|
39
|
+
- Fix deprecation warnings with redis-rb 4.8.0 [#5484]
|
40
|
+
- Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
|
41
|
+
|
42
|
+
6.5.5
|
43
|
+
----------
|
44
|
+
|
45
|
+
- Fix require issue with job_retry.rb [#5462]
|
46
|
+
- Improve Sidekiq::Web compatibility with Rack 3.x
|
47
|
+
|
48
|
+
6.5.4
|
49
|
+
----------
|
50
|
+
|
51
|
+
- Fix invalid code on Ruby 2.5 [#5460]
|
52
|
+
- Fix further metrics dependency issues [#5457]
|
53
|
+
|
54
|
+
6.5.3
|
55
|
+
----------
|
56
|
+
|
57
|
+
- Don't require metrics code without explicit opt-in [#5456]
|
58
|
+
|
59
|
+
6.5.2
|
60
|
+
----------
|
61
|
+
|
62
|
+
- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
|
63
|
+
- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
|
64
|
+
- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
|
65
|
+
- Smarter sorting of processes in /busy Web UI [#5398]
|
66
|
+
- Fix broken hamburger menu in mobile UI [#5428]
|
67
|
+
- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
|
68
|
+
[`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
|
69
|
+
|
5
70
|
6.5.1
|
6
71
|
----------
|
7
72
|
|
data/bin/sidekiqload
CHANGED
@@ -89,7 +89,7 @@ def Process.rss
|
|
89
89
|
`ps -o rss= -p #{Process.pid}`.chomp.to_i
|
90
90
|
end
|
91
91
|
|
92
|
-
iter =
|
92
|
+
iter = 10
|
93
93
|
count = 10_000
|
94
94
|
|
95
95
|
iter.times do
|
@@ -139,7 +139,7 @@ begin
|
|
139
139
|
events.clear
|
140
140
|
|
141
141
|
with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
|
142
|
-
launcher = Sidekiq::Launcher.new(Sidekiq
|
142
|
+
launcher = Sidekiq::Launcher.new(Sidekiq)
|
143
143
|
launcher.run
|
144
144
|
|
145
145
|
while readable_io = IO.select([self_read])
|
data/lib/sidekiq/api.rb
CHANGED
@@ -3,9 +3,31 @@
|
|
3
3
|
require "sidekiq"
|
4
4
|
|
5
5
|
require "zlib"
|
6
|
+
require "set"
|
6
7
|
require "base64"
|
7
8
|
|
9
|
+
if ENV["SIDEKIQ_METRICS_BETA"]
|
10
|
+
require "sidekiq/metrics/deploy"
|
11
|
+
require "sidekiq/metrics/query"
|
12
|
+
end
|
13
|
+
|
14
|
+
#
|
15
|
+
# Sidekiq's Data API provides a Ruby object model on top
|
16
|
+
# of Sidekiq's runtime data in Redis. This API should never
|
17
|
+
# be used within application code for business logic.
|
18
|
+
#
|
19
|
+
# The Sidekiq server process never uses this API: all data
|
20
|
+
# manipulation is done directly for performance reasons to
|
21
|
+
# ensure we are using Redis as efficiently as possible at
|
22
|
+
# every callsite.
|
23
|
+
#
|
24
|
+
|
8
25
|
module Sidekiq
|
26
|
+
# Retrieve runtime statistics from Redis regarding
|
27
|
+
# this Sidekiq cluster.
|
28
|
+
#
|
29
|
+
# stat = Sidekiq::Stats.new
|
30
|
+
# stat.processed
|
9
31
|
class Stats
|
10
32
|
def initialize
|
11
33
|
fetch_stats_fast!
|
@@ -52,6 +74,7 @@ module Sidekiq
|
|
52
74
|
end
|
53
75
|
|
54
76
|
# O(1) redis calls
|
77
|
+
# @api private
|
55
78
|
def fetch_stats_fast!
|
56
79
|
pipe1_res = Sidekiq.redis { |conn|
|
57
80
|
conn.pipelined do |pipeline|
|
@@ -91,6 +114,7 @@ module Sidekiq
|
|
91
114
|
end
|
92
115
|
|
93
116
|
# O(number of processes + number of queues) redis calls
|
117
|
+
# @api private
|
94
118
|
def fetch_stats_slow!
|
95
119
|
processes = Sidekiq.redis { |conn|
|
96
120
|
conn.sscan_each("processes").to_a
|
@@ -116,11 +140,13 @@ module Sidekiq
|
|
116
140
|
@stats
|
117
141
|
end
|
118
142
|
|
143
|
+
# @api private
|
119
144
|
def fetch_stats!
|
120
145
|
fetch_stats_fast!
|
121
146
|
fetch_stats_slow!
|
122
147
|
end
|
123
148
|
|
149
|
+
# @api private
|
124
150
|
def reset(*stats)
|
125
151
|
all = %w[failed processed]
|
126
152
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
@@ -202,9 +228,10 @@ module Sidekiq
|
|
202
228
|
end
|
203
229
|
|
204
230
|
##
|
205
|
-
#
|
231
|
+
# Represents a queue within Sidekiq.
|
206
232
|
# Allows enumeration of all jobs within the queue
|
207
|
-
# and deletion of jobs.
|
233
|
+
# and deletion of jobs. NB: this queue data is real-time
|
234
|
+
# and is changing within Redis moment by moment.
|
208
235
|
#
|
209
236
|
# queue = Sidekiq::Queue.new("mailer")
|
210
237
|
# queue.each do |job|
|
@@ -212,7 +239,6 @@ module Sidekiq
|
|
212
239
|
# job.args # => [1, 2, 3]
|
213
240
|
# job.delete if job.jid == 'abcdef1234567890'
|
214
241
|
# end
|
215
|
-
#
|
216
242
|
class Queue
|
217
243
|
include Enumerable
|
218
244
|
|
@@ -296,41 +322,53 @@ module Sidekiq
|
|
296
322
|
end
|
297
323
|
|
298
324
|
# delete all jobs within this queue
|
325
|
+
# @return [Boolean] true
|
299
326
|
def clear
|
300
327
|
Sidekiq.redis do |conn|
|
301
328
|
conn.multi do |transaction|
|
302
329
|
transaction.unlink(@rname)
|
303
|
-
transaction.srem("queues", name)
|
330
|
+
transaction.srem("queues", [name])
|
304
331
|
end
|
305
332
|
end
|
333
|
+
true
|
306
334
|
end
|
307
335
|
alias_method :💣, :clear
|
308
336
|
|
309
|
-
|
337
|
+
# :nodoc:
|
338
|
+
# @api private
|
339
|
+
def as_json(options = nil)
|
310
340
|
{name: name} # 5336
|
311
341
|
end
|
312
342
|
end
|
313
343
|
|
314
344
|
##
|
315
|
-
#
|
316
|
-
# sorted set.
|
345
|
+
# Represents a pending job within a Sidekiq queue.
|
317
346
|
#
|
318
347
|
# The job should be considered immutable but may be
|
319
348
|
# removed from the queue via JobRecord#delete.
|
320
|
-
#
|
321
349
|
class JobRecord
|
350
|
+
# the parsed Hash of job data
|
351
|
+
# @!attribute [r] Item
|
322
352
|
attr_reader :item
|
353
|
+
# the underlying String in Redis
|
354
|
+
# @!attribute [r] Value
|
323
355
|
attr_reader :value
|
356
|
+
# the queue associated with this job
|
357
|
+
# @!attribute [r] Queue
|
324
358
|
attr_reader :queue
|
325
359
|
|
326
|
-
|
360
|
+
# :nodoc:
|
361
|
+
# @api private
|
362
|
+
def initialize(item, queue_name = nil)
|
327
363
|
@args = nil
|
328
364
|
@value = item
|
329
365
|
@item = item.is_a?(Hash) ? item : parse(item)
|
330
366
|
@queue = queue_name || @item["queue"]
|
331
367
|
end
|
332
368
|
|
333
|
-
|
369
|
+
# :nodoc:
|
370
|
+
# @api private
|
371
|
+
def parse(item)
|
334
372
|
Sidekiq.load_json(item)
|
335
373
|
rescue JSON::ParserError
|
336
374
|
# If the job payload in Redis is invalid JSON, we'll load
|
@@ -341,6 +379,8 @@ module Sidekiq
|
|
341
379
|
{}
|
342
380
|
end
|
343
381
|
|
382
|
+
# This is the job class which Sidekiq will execute. If using ActiveJob,
|
383
|
+
# this class will be the ActiveJob adapter class rather than a specific job.
|
344
384
|
def klass
|
345
385
|
self["class"]
|
346
386
|
end
|
@@ -457,7 +497,7 @@ module Sidekiq
|
|
457
497
|
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
458
498
|
# memory yet so the YAML can't be loaded.
|
459
499
|
# TODO is this still necessary? Zeitwerk reloader should handle?
|
460
|
-
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.
|
500
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
461
501
|
default
|
462
502
|
end
|
463
503
|
|
@@ -480,21 +520,27 @@ module Sidekiq
|
|
480
520
|
end
|
481
521
|
|
482
522
|
# Represents a job within a Redis sorted set where the score
|
483
|
-
# represents a timestamp
|
523
|
+
# represents a timestamp associated with the job. This timestamp
|
524
|
+
# could be the scheduled time for it to run (e.g. scheduled set),
|
525
|
+
# or the expiration date after which the entry should be deleted (e.g. dead set).
|
484
526
|
class SortedEntry < JobRecord
|
485
527
|
attr_reader :score
|
486
528
|
attr_reader :parent
|
487
529
|
|
488
|
-
|
530
|
+
# :nodoc:
|
531
|
+
# @api private
|
532
|
+
def initialize(parent, score, item)
|
489
533
|
super(item)
|
490
534
|
@score = Float(score)
|
491
535
|
@parent = parent
|
492
536
|
end
|
493
537
|
|
538
|
+
# The timestamp associated with this entry
|
494
539
|
def at
|
495
540
|
Time.at(score).utc
|
496
541
|
end
|
497
542
|
|
543
|
+
# remove this entry from the sorted set
|
498
544
|
def delete
|
499
545
|
if @value
|
500
546
|
@parent.delete_by_value(@parent.name, @value)
|
@@ -505,7 +551,7 @@ module Sidekiq
|
|
505
551
|
|
506
552
|
# Change the scheduled time for this job.
|
507
553
|
#
|
508
|
-
# @param [Time] the new timestamp
|
554
|
+
# @param at [Time] the new timestamp for this job
|
509
555
|
def reschedule(at)
|
510
556
|
Sidekiq.redis do |conn|
|
511
557
|
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
@@ -579,20 +625,32 @@ module Sidekiq
|
|
579
625
|
end
|
580
626
|
end
|
581
627
|
|
628
|
+
# Base class for all sorted sets within Sidekiq.
|
582
629
|
class SortedSet
|
583
630
|
include Enumerable
|
584
631
|
|
632
|
+
# Redis key of the set
|
633
|
+
# @!attribute [r] Name
|
585
634
|
attr_reader :name
|
586
635
|
|
636
|
+
# :nodoc:
|
637
|
+
# @api private
|
587
638
|
def initialize(name)
|
588
639
|
@name = name
|
589
640
|
@_size = size
|
590
641
|
end
|
591
642
|
|
643
|
+
# real-time size of the set, will change
|
592
644
|
def size
|
593
645
|
Sidekiq.redis { |c| c.zcard(name) }
|
594
646
|
end
|
595
647
|
|
648
|
+
# Scan through each element of the sorted set, yielding each to the supplied block.
|
649
|
+
# Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
|
650
|
+
#
|
651
|
+
# @param match [String] a snippet or regexp to filter matches.
|
652
|
+
# @param count [Integer] number of elements to retrieve at a time, default 100
|
653
|
+
# @yieldparam [Sidekiq::SortedEntry] each entry
|
596
654
|
def scan(match, count = 100)
|
597
655
|
return to_enum(:scan, match, count) unless block_given?
|
598
656
|
|
@@ -604,22 +662,32 @@ module Sidekiq
|
|
604
662
|
end
|
605
663
|
end
|
606
664
|
|
665
|
+
# @return [Boolean] always true
|
607
666
|
def clear
|
608
667
|
Sidekiq.redis do |conn|
|
609
668
|
conn.unlink(name)
|
610
669
|
end
|
670
|
+
true
|
611
671
|
end
|
612
672
|
alias_method :💣, :clear
|
613
673
|
|
614
|
-
|
674
|
+
# :nodoc:
|
675
|
+
# @api private
|
676
|
+
def as_json(options = nil)
|
615
677
|
{name: name} # 5336
|
616
678
|
end
|
617
679
|
end
|
618
680
|
|
681
|
+
# Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
|
682
|
+
# Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
|
683
|
+
# e.g. Batches.
|
619
684
|
class JobSet < SortedSet
|
620
|
-
|
685
|
+
# Add a job with the associated timestamp to this set.
|
686
|
+
# @param timestamp [Time] the score for the job
|
687
|
+
# @param job [Hash] the job data
|
688
|
+
def schedule(timestamp, job)
|
621
689
|
Sidekiq.redis do |conn|
|
622
|
-
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(
|
690
|
+
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
|
623
691
|
end
|
624
692
|
end
|
625
693
|
|
@@ -647,6 +715,10 @@ module Sidekiq
|
|
647
715
|
##
|
648
716
|
# Fetch jobs that match a given time or Range. Job ID is an
|
649
717
|
# optional second argument.
|
718
|
+
#
|
719
|
+
# @param score [Time,Range] a specific timestamp or range
|
720
|
+
# @param jid [String, optional] find a specific JID within the score
|
721
|
+
# @return [Array<SortedEntry>] any results found, can be empty
|
650
722
|
def fetch(score, jid = nil)
|
651
723
|
begin_score, end_score =
|
652
724
|
if score.is_a?(Range)
|
@@ -668,7 +740,10 @@ module Sidekiq
|
|
668
740
|
|
669
741
|
##
|
670
742
|
# Find the job with the given JID within this sorted set.
|
671
|
-
# This is a
|
743
|
+
# *This is a slow O(n) operation*. Do not use for app logic.
|
744
|
+
#
|
745
|
+
# @param jid [String] the job identifier
|
746
|
+
# @return [SortedEntry] the record or nil
|
672
747
|
def find_job(jid)
|
673
748
|
Sidekiq.redis do |conn|
|
674
749
|
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
@@ -680,6 +755,8 @@ module Sidekiq
|
|
680
755
|
nil
|
681
756
|
end
|
682
757
|
|
758
|
+
# :nodoc:
|
759
|
+
# @api private
|
683
760
|
def delete_by_value(name, value)
|
684
761
|
Sidekiq.redis do |conn|
|
685
762
|
ret = conn.zrem(name, value)
|
@@ -688,6 +765,8 @@ module Sidekiq
|
|
688
765
|
end
|
689
766
|
end
|
690
767
|
|
768
|
+
# :nodoc:
|
769
|
+
# @api private
|
691
770
|
def delete_by_jid(score, jid)
|
692
771
|
Sidekiq.redis do |conn|
|
693
772
|
elements = conn.zrangebyscore(name, score, score)
|
@@ -708,10 +787,10 @@ module Sidekiq
|
|
708
787
|
end
|
709
788
|
|
710
789
|
##
|
711
|
-
#
|
790
|
+
# The set of scheduled jobs within Sidekiq.
|
712
791
|
# Based on this, you can search/filter for jobs. Here's an
|
713
|
-
# example where I'm selecting
|
714
|
-
# and deleting them from the
|
792
|
+
# example where I'm selecting jobs based on some complex logic
|
793
|
+
# and deleting them from the scheduled set.
|
715
794
|
#
|
716
795
|
# r = Sidekiq::ScheduledSet.new
|
717
796
|
# r.select do |scheduled|
|
@@ -726,7 +805,7 @@ module Sidekiq
|
|
726
805
|
end
|
727
806
|
|
728
807
|
##
|
729
|
-
#
|
808
|
+
# The set of retries within Sidekiq.
|
730
809
|
# Based on this, you can search/filter for jobs. Here's an
|
731
810
|
# example where I'm selecting all jobs of a certain type
|
732
811
|
# and deleting them from the retry queue.
|
@@ -742,23 +821,29 @@ module Sidekiq
|
|
742
821
|
super "retry"
|
743
822
|
end
|
744
823
|
|
824
|
+
# Enqueues all jobs pending within the retry set.
|
745
825
|
def retry_all
|
746
826
|
each(&:retry) while size > 0
|
747
827
|
end
|
748
828
|
|
829
|
+
# Kills all jobs pending within the retry set.
|
749
830
|
def kill_all
|
750
831
|
each(&:kill) while size > 0
|
751
832
|
end
|
752
833
|
end
|
753
834
|
|
754
835
|
##
|
755
|
-
#
|
836
|
+
# The set of dead jobs within Sidekiq. Dead jobs have failed all of
|
837
|
+
# their retries and are helding in this set pending some sort of manual
|
838
|
+
# fix. They will be removed after 6 months (dead_timeout) if not.
|
756
839
|
#
|
757
840
|
class DeadSet < JobSet
|
758
841
|
def initialize
|
759
842
|
super "dead"
|
760
843
|
end
|
761
844
|
|
845
|
+
# Add the given job to the Dead set.
|
846
|
+
# @param message [String] the job data as JSON
|
762
847
|
def kill(message, opts = {})
|
763
848
|
now = Time.now.to_f
|
764
849
|
Sidekiq.redis do |conn|
|
@@ -780,14 +865,19 @@ module Sidekiq
|
|
780
865
|
true
|
781
866
|
end
|
782
867
|
|
868
|
+
# Enqueue all dead jobs
|
783
869
|
def retry_all
|
784
870
|
each(&:retry) while size > 0
|
785
871
|
end
|
786
872
|
|
873
|
+
# The maximum size of the Dead set. Older entries will be trimmed
|
874
|
+
# to stay within this limit. Default value is 10,000.
|
787
875
|
def self.max_jobs
|
788
876
|
Sidekiq[:dead_max_jobs]
|
789
877
|
end
|
790
878
|
|
879
|
+
# The time limit for entries within the Dead set. Older entries will be thrown away.
|
880
|
+
# Default value is six months.
|
791
881
|
def self.timeout
|
792
882
|
Sidekiq[:dead_timeout_in_seconds]
|
793
883
|
end
|
@@ -798,21 +888,28 @@ module Sidekiq
|
|
798
888
|
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
799
889
|
# so this set should be relatively accurate, barring network partitions.
|
800
890
|
#
|
801
|
-
#
|
891
|
+
# @yieldparam [Sidekiq::Process]
|
802
892
|
#
|
803
893
|
class ProcessSet
|
804
894
|
include Enumerable
|
805
895
|
|
896
|
+
# :nodoc:
|
897
|
+
# @api private
|
806
898
|
def initialize(clean_plz = true)
|
807
899
|
cleanup if clean_plz
|
808
900
|
end
|
809
901
|
|
810
902
|
# Cleans up dead processes recorded in Redis.
|
811
903
|
# Returns the number of processes cleaned.
|
904
|
+
# :nodoc:
|
905
|
+
# @api private
|
812
906
|
def cleanup
|
907
|
+
# dont run cleanup more than once per minute
|
908
|
+
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
|
909
|
+
|
813
910
|
count = 0
|
814
911
|
Sidekiq.redis do |conn|
|
815
|
-
procs = conn.sscan_each("processes").to_a
|
912
|
+
procs = conn.sscan_each("processes").to_a
|
816
913
|
heartbeats = conn.pipelined { |pipeline|
|
817
914
|
procs.each do |key|
|
818
915
|
pipeline.hget(key, "info")
|
@@ -863,6 +960,7 @@ module Sidekiq
|
|
863
960
|
# based on current heartbeat. #each does that and ensures the set only
|
864
961
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
865
962
|
# 60 seconds.
|
963
|
+
# @return [Integer] current number of registered Sidekiq processes
|
866
964
|
def size
|
867
965
|
Sidekiq.redis { |conn| conn.scard("processes") }
|
868
966
|
end
|
@@ -870,10 +968,12 @@ module Sidekiq
|
|
870
968
|
# Total number of threads available to execute jobs.
|
871
969
|
# For Sidekiq Enterprise customers this number (in production) must be
|
872
970
|
# less than or equal to your licensed concurrency.
|
971
|
+
# @return [Integer] the sum of process concurrency
|
873
972
|
def total_concurrency
|
874
973
|
sum { |x| x["concurrency"].to_i }
|
875
974
|
end
|
876
975
|
|
976
|
+
# @return [Integer] total amount of RSS memory consumed by Sidekiq processes
|
877
977
|
def total_rss_in_kb
|
878
978
|
sum { |x| x["rss"].to_i }
|
879
979
|
end
|
@@ -882,6 +982,8 @@ module Sidekiq
|
|
882
982
|
# Returns the identity of the current cluster leader or "" if no leader.
|
883
983
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
884
984
|
# or Sidekiq Pro.
|
985
|
+
# @return [String] Identity of cluster leader
|
986
|
+
# @return [String] empty string if no leader
|
885
987
|
def leader
|
886
988
|
@leader ||= begin
|
887
989
|
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
@@ -908,6 +1010,8 @@ module Sidekiq
|
|
908
1010
|
# 'identity' => <unique string identifying the process>,
|
909
1011
|
# }
|
910
1012
|
class Process
|
1013
|
+
# :nodoc:
|
1014
|
+
# @api private
|
911
1015
|
def initialize(hash)
|
912
1016
|
@attribs = hash
|
913
1017
|
end
|
@@ -932,18 +1036,31 @@ module Sidekiq
|
|
932
1036
|
self["queues"]
|
933
1037
|
end
|
934
1038
|
|
1039
|
+
# Signal this process to stop processing new jobs.
|
1040
|
+
# It will continue to execute jobs it has already fetched.
|
1041
|
+
# This method is *asynchronous* and it can take 5-10
|
1042
|
+
# seconds for the process to quiet.
|
935
1043
|
def quiet!
|
936
1044
|
signal("TSTP")
|
937
1045
|
end
|
938
1046
|
|
1047
|
+
# Signal this process to shutdown.
|
1048
|
+
# It will shutdown within its configured :timeout value, default 25 seconds.
|
1049
|
+
# This method is *asynchronous* and it can take 5-10
|
1050
|
+
# seconds for the process to start shutting down.
|
939
1051
|
def stop!
|
940
1052
|
signal("TERM")
|
941
1053
|
end
|
942
1054
|
|
1055
|
+
# Signal this process to log backtraces for all threads.
|
1056
|
+
# Useful if you have a frozen or deadlocked process which is
|
1057
|
+
# still sending a heartbeat.
|
1058
|
+
# This method is *asynchronous* and it can take 5-10 seconds.
|
943
1059
|
def dump_threads
|
944
1060
|
signal("TTIN")
|
945
1061
|
end
|
946
1062
|
|
1063
|
+
# @return [Boolean] true if this process is quiet or shutting down
|
947
1064
|
def stopping?
|
948
1065
|
self["quiet"] == "true"
|
949
1066
|
end
|
@@ -986,24 +1103,31 @@ module Sidekiq
|
|
986
1103
|
|
987
1104
|
def each(&block)
|
988
1105
|
results = []
|
1106
|
+
procs = nil
|
1107
|
+
all_works = nil
|
1108
|
+
|
989
1109
|
Sidekiq.redis do |conn|
|
990
|
-
procs = conn.sscan_each("processes").to_a
|
991
|
-
|
992
|
-
|
993
|
-
|
1110
|
+
procs = conn.sscan_each("processes").to_a.sort
|
1111
|
+
|
1112
|
+
all_works = conn.pipelined do |pipeline|
|
1113
|
+
procs.each do |key|
|
994
1114
|
pipeline.hgetall("#{key}:work")
|
995
|
-
}
|
996
|
-
next unless valid
|
997
|
-
workers.each_pair do |tid, json|
|
998
|
-
hsh = Sidekiq.load_json(json)
|
999
|
-
p = hsh["payload"]
|
1000
|
-
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
1001
|
-
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
1002
|
-
results << [key, tid, hsh]
|
1003
1115
|
end
|
1004
1116
|
end
|
1005
1117
|
end
|
1006
1118
|
|
1119
|
+
procs.zip(all_works).each do |key, workers|
|
1120
|
+
workers.each_pair do |tid, json|
|
1121
|
+
next if json.empty?
|
1122
|
+
|
1123
|
+
hsh = Sidekiq.load_json(json)
|
1124
|
+
p = hsh["payload"]
|
1125
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
1126
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
1127
|
+
results << [key, tid, hsh]
|
1128
|
+
end
|
1129
|
+
end
|
1130
|
+
|
1007
1131
|
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
1008
1132
|
end
|
1009
1133
|
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -12,6 +12,17 @@ require "sidekiq"
|
|
12
12
|
require "sidekiq/component"
|
13
13
|
require "sidekiq/launcher"
|
14
14
|
|
15
|
+
# module ScoutApm
|
16
|
+
# VERSION = "5.3.1"
|
17
|
+
# end
|
18
|
+
fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
|
19
|
+
|
20
|
+
|
21
|
+
scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
|
22
|
+
|
23
|
+
|
24
|
+
EOM
|
25
|
+
|
15
26
|
module Sidekiq # :nodoc:
|
16
27
|
class CLI
|
17
28
|
include Sidekiq::Component
|
@@ -214,6 +225,7 @@ module Sidekiq # :nodoc:
|
|
214
225
|
# Both Sinatra 2.0+ and Sidekiq support this term.
|
215
226
|
# RAILS_ENV and RACK_ENV are there for legacy support.
|
216
227
|
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
228
|
+
config[:environment] = @environment
|
217
229
|
end
|
218
230
|
|
219
231
|
def symbolize_keys_deep!(hash)
|
@@ -426,3 +438,4 @@ module Sidekiq # :nodoc:
|
|
426
438
|
end
|
427
439
|
|
428
440
|
require "sidekiq/systemd"
|
441
|
+
require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
|
data/lib/sidekiq/client.rb
CHANGED
@@ -176,7 +176,7 @@ module Sidekiq
|
|
176
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
177
177
|
int = interval.to_f
|
178
178
|
now = Time.now.to_f
|
179
|
-
ts = (int < 1_000_000_000 ? now + int : int)
|
179
|
+
ts = ((int < 1_000_000_000) ? now + int : int)
|
180
180
|
|
181
181
|
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
182
182
|
item.delete("at") if ts <= now
|
@@ -231,7 +231,7 @@ module Sidekiq
|
|
231
231
|
entry["enqueued_at"] = now
|
232
232
|
Sidekiq.dump_json(entry)
|
233
233
|
}
|
234
|
-
conn.sadd("queues", queue)
|
234
|
+
conn.sadd("queues", [queue])
|
235
235
|
conn.lpush("queue:#{queue}", to_push)
|
236
236
|
end
|
237
237
|
end
|
data/lib/sidekiq/component.rb
CHANGED
@@ -47,6 +47,7 @@ module Sidekiq
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def fire_event(event, options = {})
|
50
|
+
oneshot = options.fetch(:oneshot, true)
|
50
51
|
reverse = options[:reverse]
|
51
52
|
reraise = options[:reraise]
|
52
53
|
|
@@ -58,7 +59,7 @@ module Sidekiq
|
|
58
59
|
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
59
60
|
raise ex if reraise
|
60
61
|
end
|
61
|
-
arr.clear # once we've fired an event, we never fire it again
|
62
|
+
arr.clear if oneshot # once we've fired an event, we never fire it again
|
62
63
|
end
|
63
64
|
end
|
64
65
|
end
|