sidekiq 6.5.1 → 6.5.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +65 -0
  3. data/bin/sidekiqload +2 -2
  4. data/lib/sidekiq/api.rb +161 -37
  5. data/lib/sidekiq/cli.rb +13 -0
  6. data/lib/sidekiq/client.rb +2 -2
  7. data/lib/sidekiq/component.rb +2 -1
  8. data/lib/sidekiq/fetch.rb +2 -2
  9. data/lib/sidekiq/job_retry.rb +55 -35
  10. data/lib/sidekiq/launcher.rb +6 -4
  11. data/lib/sidekiq/metrics/deploy.rb +47 -0
  12. data/lib/sidekiq/metrics/query.rb +153 -0
  13. data/lib/sidekiq/metrics/shared.rb +94 -0
  14. data/lib/sidekiq/metrics/tracking.rb +134 -0
  15. data/lib/sidekiq/middleware/chain.rb +70 -35
  16. data/lib/sidekiq/middleware/current_attributes.rb +14 -12
  17. data/lib/sidekiq/monitor.rb +1 -1
  18. data/lib/sidekiq/paginator.rb +9 -1
  19. data/lib/sidekiq/processor.rb +9 -3
  20. data/lib/sidekiq/rails.rb +10 -11
  21. data/lib/sidekiq/redis_connection.rb +0 -2
  22. data/lib/sidekiq/scheduled.rb +43 -15
  23. data/lib/sidekiq/version.rb +1 -1
  24. data/lib/sidekiq/web/action.rb +3 -3
  25. data/lib/sidekiq/web/application.rb +21 -5
  26. data/lib/sidekiq/web/helpers.rb +17 -4
  27. data/lib/sidekiq/web.rb +5 -1
  28. data/lib/sidekiq/worker.rb +6 -3
  29. data/lib/sidekiq.rb +9 -1
  30. data/sidekiq.gemspec +2 -2
  31. data/web/assets/javascripts/application.js +2 -1
  32. data/web/assets/javascripts/chart.min.js +13 -0
  33. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  34. data/web/assets/javascripts/dashboard.js +0 -17
  35. data/web/assets/javascripts/graph.js +16 -0
  36. data/web/assets/javascripts/metrics.js +262 -0
  37. data/web/assets/stylesheets/application.css +44 -1
  38. data/web/locales/el.yml +43 -19
  39. data/web/locales/en.yml +7 -0
  40. data/web/locales/ja.yml +7 -0
  41. data/web/locales/zh-cn.yml +36 -11
  42. data/web/locales/zh-tw.yml +32 -7
  43. data/web/views/_nav.erb +1 -1
  44. data/web/views/busy.erb +7 -2
  45. data/web/views/dashboard.erb +1 -0
  46. data/web/views/metrics.erb +69 -0
  47. data/web/views/metrics_for_job.erb +87 -0
  48. data/web/views/queue.erb +5 -1
  49. metadata +29 -8
  50. data/lib/sidekiq/.DS_Store +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1afbc6a1a0b14403e9148e746c08e0a2b24e634fca05288982c96719675607de
4
- data.tar.gz: 3ff3f8df76b565f42030462eb8d09b673b89751dcd6a0b7c41a255789960d321
3
+ metadata.gz: 55c6f9a8c0f2810bcbe7744c95204893d73f534666f6091c74dcb5199e7a4a28
4
+ data.tar.gz: c42690ef0d1876c94eb51fb68319275d11f10169826180db921b34d6334a1a54
5
5
  SHA512:
6
- metadata.gz: e8a68611735322d98cc517f1d03ef02394497f8eb505e0db496909cac1f6b7f0179f38ce1966546f717115395102c78fdad9ae0fd947360307b9288dcc22b369
7
- data.tar.gz: 163e41dfb153a4e2ec50d407bde08cb8a395909807053f24478a73af1a6a9858b5aec92cbf618d4d066560a2b7bedaa2f88b48f823609e2d78f21320270cb97e
6
+ metadata.gz: 461b559e18cbdf8fe6ba20ab9fa38d08fd3b7b8d14dce7a7f9369d678e92bd25c07734bc14b0167f83589c0f03501897195b3fdf9cfd5de5a60f039bf7436f98
7
+ data.tar.gz: ce0490b438a22a71c37e5a65193a7728e3297b437730d30a3653807ff1e89476db1f8c62d8de50c0c8cebbbce2ba6fb65f110855e071c62746fe8697a0f636e1
data/Changes.md CHANGED
@@ -2,6 +2,71 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
+ 6.5.11
6
+ ----------
7
+
8
+ - Fix for Rails 7.1 [#6067]
9
+
10
+ 6.5.10
11
+ ----------
12
+
13
+ - Web UI DoS vector [#6045] CVE-2023-26141
14
+ - Fix broadcast logger with Rails 7.1 [#6054]
15
+
16
+ 6.5.9
17
+ ----------
18
+
19
+ - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
20
+
21
+ 6.5.8
22
+ ----------
23
+
24
+ - Fail if using a bad version of scout_apm [#5616]
25
+ - Add pagination to Busy page [#5556]
26
+ - Speed up WorkSet#each [#5559]
27
+ - Adjust CurrentAttributes to work with the String class name so we aren't referencing
28
+ the Class within a Rails initializer [#5536]
29
+
30
+ 6.5.7
31
+ ----------
32
+
33
+ - Updates for JA and ZH locales
34
+ - Further optimizations for scheduled polling [#5513]
35
+
36
+ 6.5.6
37
+ ----------
38
+
39
+ - Fix deprecation warnings with redis-rb 4.8.0 [#5484]
40
+ - Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
41
+
42
+ 6.5.5
43
+ ----------
44
+
45
+ - Fix require issue with job_retry.rb [#5462]
46
+ - Improve Sidekiq::Web compatibility with Rack 3.x
47
+
48
+ 6.5.4
49
+ ----------
50
+
51
+ - Fix invalid code on Ruby 2.5 [#5460]
52
+ - Fix further metrics dependency issues [#5457]
53
+
54
+ 6.5.3
55
+ ----------
56
+
57
+ - Don't require metrics code without explicit opt-in [#5456]
58
+
59
+ 6.5.2
60
+ ----------
61
+
62
+ - [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
63
+ - Add `Context` column on queue page which shows any CurrentAttributes [#5450]
64
+ - `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
65
+ - Smarter sorting of processes in /busy Web UI [#5398]
66
+ - Fix broken hamburger menu in mobile UI [#5428]
67
+ - Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
68
+ [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
69
+
5
70
  6.5.1
6
71
  ----------
7
72
 
data/bin/sidekiqload CHANGED
@@ -89,7 +89,7 @@ def Process.rss
89
89
  `ps -o rss= -p #{Process.pid}`.chomp.to_i
90
90
  end
91
91
 
92
- iter = 50
92
+ iter = 10
93
93
  count = 10_000
94
94
 
95
95
  iter.times do
@@ -139,7 +139,7 @@ begin
139
139
  events.clear
140
140
 
141
141
  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
142
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
142
+ launcher = Sidekiq::Launcher.new(Sidekiq)
143
143
  launcher.run
144
144
 
145
145
  while readable_io = IO.select([self_read])
data/lib/sidekiq/api.rb CHANGED
@@ -3,9 +3,31 @@
3
3
  require "sidekiq"
4
4
 
5
5
  require "zlib"
6
+ require "set"
6
7
  require "base64"
7
8
 
9
+ if ENV["SIDEKIQ_METRICS_BETA"]
10
+ require "sidekiq/metrics/deploy"
11
+ require "sidekiq/metrics/query"
12
+ end
13
+
14
+ #
15
+ # Sidekiq's Data API provides a Ruby object model on top
16
+ # of Sidekiq's runtime data in Redis. This API should never
17
+ # be used within application code for business logic.
18
+ #
19
+ # The Sidekiq server process never uses this API: all data
20
+ # manipulation is done directly for performance reasons to
21
+ # ensure we are using Redis as efficiently as possible at
22
+ # every callsite.
23
+ #
24
+
8
25
  module Sidekiq
26
+ # Retrieve runtime statistics from Redis regarding
27
+ # this Sidekiq cluster.
28
+ #
29
+ # stat = Sidekiq::Stats.new
30
+ # stat.processed
9
31
  class Stats
10
32
  def initialize
11
33
  fetch_stats_fast!
@@ -52,6 +74,7 @@ module Sidekiq
52
74
  end
53
75
 
54
76
  # O(1) redis calls
77
+ # @api private
55
78
  def fetch_stats_fast!
56
79
  pipe1_res = Sidekiq.redis { |conn|
57
80
  conn.pipelined do |pipeline|
@@ -91,6 +114,7 @@ module Sidekiq
91
114
  end
92
115
 
93
116
  # O(number of processes + number of queues) redis calls
117
+ # @api private
94
118
  def fetch_stats_slow!
95
119
  processes = Sidekiq.redis { |conn|
96
120
  conn.sscan_each("processes").to_a
@@ -116,11 +140,13 @@ module Sidekiq
116
140
  @stats
117
141
  end
118
142
 
143
+ # @api private
119
144
  def fetch_stats!
120
145
  fetch_stats_fast!
121
146
  fetch_stats_slow!
122
147
  end
123
148
 
149
+ # @api private
124
150
  def reset(*stats)
125
151
  all = %w[failed processed]
126
152
  stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
@@ -202,9 +228,10 @@ module Sidekiq
202
228
  end
203
229
 
204
230
  ##
205
- # Encapsulates a queue within Sidekiq.
231
+ # Represents a queue within Sidekiq.
206
232
  # Allows enumeration of all jobs within the queue
207
- # and deletion of jobs.
233
+ # and deletion of jobs. NB: this queue data is real-time
234
+ # and is changing within Redis moment by moment.
208
235
  #
209
236
  # queue = Sidekiq::Queue.new("mailer")
210
237
  # queue.each do |job|
@@ -212,7 +239,6 @@ module Sidekiq
212
239
  # job.args # => [1, 2, 3]
213
240
  # job.delete if job.jid == 'abcdef1234567890'
214
241
  # end
215
- #
216
242
  class Queue
217
243
  include Enumerable
218
244
 
@@ -296,41 +322,53 @@ module Sidekiq
296
322
  end
297
323
 
298
324
  # delete all jobs within this queue
325
+ # @return [Boolean] true
299
326
  def clear
300
327
  Sidekiq.redis do |conn|
301
328
  conn.multi do |transaction|
302
329
  transaction.unlink(@rname)
303
- transaction.srem("queues", name)
330
+ transaction.srem("queues", [name])
304
331
  end
305
332
  end
333
+ true
306
334
  end
307
335
  alias_method :💣, :clear
308
336
 
309
- def as_json(options = nil) # :nodoc:
337
+ # :nodoc:
338
+ # @api private
339
+ def as_json(options = nil)
310
340
  {name: name} # 5336
311
341
  end
312
342
  end
313
343
 
314
344
  ##
315
- # Encapsulates a pending job within a Sidekiq queue or
316
- # sorted set.
345
+ # Represents a pending job within a Sidekiq queue.
317
346
  #
318
347
  # The job should be considered immutable but may be
319
348
  # removed from the queue via JobRecord#delete.
320
- #
321
349
  class JobRecord
350
+ # the parsed Hash of job data
351
+ # @!attribute [r] Item
322
352
  attr_reader :item
353
+ # the underlying String in Redis
354
+ # @!attribute [r] Value
323
355
  attr_reader :value
356
+ # the queue associated with this job
357
+ # @!attribute [r] Queue
324
358
  attr_reader :queue
325
359
 
326
- def initialize(item, queue_name = nil) # :nodoc:
360
+ # :nodoc:
361
+ # @api private
362
+ def initialize(item, queue_name = nil)
327
363
  @args = nil
328
364
  @value = item
329
365
  @item = item.is_a?(Hash) ? item : parse(item)
330
366
  @queue = queue_name || @item["queue"]
331
367
  end
332
368
 
333
- def parse(item) # :nodoc:
369
+ # :nodoc:
370
+ # @api private
371
+ def parse(item)
334
372
  Sidekiq.load_json(item)
335
373
  rescue JSON::ParserError
336
374
  # If the job payload in Redis is invalid JSON, we'll load
@@ -341,6 +379,8 @@ module Sidekiq
341
379
  {}
342
380
  end
343
381
 
382
+ # This is the job class which Sidekiq will execute. If using ActiveJob,
383
+ # this class will be the ActiveJob adapter class rather than a specific job.
344
384
  def klass
345
385
  self["class"]
346
386
  end
@@ -457,7 +497,7 @@ module Sidekiq
457
497
  # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
458
498
  # memory yet so the YAML can't be loaded.
459
499
  # TODO is this still necessary? Zeitwerk reloader should handle?
460
- Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
500
+ Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
461
501
  default
462
502
  end
463
503
 
@@ -480,21 +520,27 @@ module Sidekiq
480
520
  end
481
521
 
482
522
  # Represents a job within a Redis sorted set where the score
483
- # represents a timestamp for the job.
523
+ # represents a timestamp associated with the job. This timestamp
524
+ # could be the scheduled time for it to run (e.g. scheduled set),
525
+ # or the expiration date after which the entry should be deleted (e.g. dead set).
484
526
  class SortedEntry < JobRecord
485
527
  attr_reader :score
486
528
  attr_reader :parent
487
529
 
488
- def initialize(parent, score, item) # :nodoc:
530
+ # :nodoc:
531
+ # @api private
532
+ def initialize(parent, score, item)
489
533
  super(item)
490
534
  @score = Float(score)
491
535
  @parent = parent
492
536
  end
493
537
 
538
+ # The timestamp associated with this entry
494
539
  def at
495
540
  Time.at(score).utc
496
541
  end
497
542
 
543
+ # remove this entry from the sorted set
498
544
  def delete
499
545
  if @value
500
546
  @parent.delete_by_value(@parent.name, @value)
@@ -505,7 +551,7 @@ module Sidekiq
505
551
 
506
552
  # Change the scheduled time for this job.
507
553
  #
508
- # @param [Time] the new timestamp when this job will be enqueued.
554
+ # @param at [Time] the new timestamp for this job
509
555
  def reschedule(at)
510
556
  Sidekiq.redis do |conn|
511
557
  conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
@@ -579,20 +625,32 @@ module Sidekiq
579
625
  end
580
626
  end
581
627
 
628
+ # Base class for all sorted sets within Sidekiq.
582
629
  class SortedSet
583
630
  include Enumerable
584
631
 
632
+ # Redis key of the set
633
+ # @!attribute [r] Name
585
634
  attr_reader :name
586
635
 
636
+ # :nodoc:
637
+ # @api private
587
638
  def initialize(name)
588
639
  @name = name
589
640
  @_size = size
590
641
  end
591
642
 
643
+ # real-time size of the set, will change
592
644
  def size
593
645
  Sidekiq.redis { |c| c.zcard(name) }
594
646
  end
595
647
 
648
+ # Scan through each element of the sorted set, yielding each to the supplied block.
649
+ # Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
650
+ #
651
+ # @param match [String] a snippet or regexp to filter matches.
652
+ # @param count [Integer] number of elements to retrieve at a time, default 100
653
+ # @yieldparam [Sidekiq::SortedEntry] each entry
596
654
  def scan(match, count = 100)
597
655
  return to_enum(:scan, match, count) unless block_given?
598
656
 
@@ -604,22 +662,32 @@ module Sidekiq
604
662
  end
605
663
  end
606
664
 
665
+ # @return [Boolean] always true
607
666
  def clear
608
667
  Sidekiq.redis do |conn|
609
668
  conn.unlink(name)
610
669
  end
670
+ true
611
671
  end
612
672
  alias_method :💣, :clear
613
673
 
614
- def as_json(options = nil) # :nodoc:
674
+ # :nodoc:
675
+ # @api private
676
+ def as_json(options = nil)
615
677
  {name: name} # 5336
616
678
  end
617
679
  end
618
680
 
681
+ # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
682
+ # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
683
+ # e.g. Batches.
619
684
  class JobSet < SortedSet
620
- def schedule(timestamp, message)
685
+ # Add a job with the associated timestamp to this set.
686
+ # @param timestamp [Time] the score for the job
687
+ # @param job [Hash] the job data
688
+ def schedule(timestamp, job)
621
689
  Sidekiq.redis do |conn|
622
- conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
690
+ conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
623
691
  end
624
692
  end
625
693
 
@@ -647,6 +715,10 @@ module Sidekiq
647
715
  ##
648
716
  # Fetch jobs that match a given time or Range. Job ID is an
649
717
  # optional second argument.
718
+ #
719
+ # @param score [Time,Range] a specific timestamp or range
720
+ # @param jid [String, optional] find a specific JID within the score
721
+ # @return [Array<SortedEntry>] any results found, can be empty
650
722
  def fetch(score, jid = nil)
651
723
  begin_score, end_score =
652
724
  if score.is_a?(Range)
@@ -668,7 +740,10 @@ module Sidekiq
668
740
 
669
741
  ##
670
742
  # Find the job with the given JID within this sorted set.
671
- # This is a slower O(n) operation. Do not use for app logic.
743
+ # *This is a slow O(n) operation*. Do not use for app logic.
744
+ #
745
+ # @param jid [String] the job identifier
746
+ # @return [SortedEntry] the record or nil
672
747
  def find_job(jid)
673
748
  Sidekiq.redis do |conn|
674
749
  conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
@@ -680,6 +755,8 @@ module Sidekiq
680
755
  nil
681
756
  end
682
757
 
758
+ # :nodoc:
759
+ # @api private
683
760
  def delete_by_value(name, value)
684
761
  Sidekiq.redis do |conn|
685
762
  ret = conn.zrem(name, value)
@@ -688,6 +765,8 @@ module Sidekiq
688
765
  end
689
766
  end
690
767
 
768
+ # :nodoc:
769
+ # @api private
691
770
  def delete_by_jid(score, jid)
692
771
  Sidekiq.redis do |conn|
693
772
  elements = conn.zrangebyscore(name, score, score)
@@ -708,10 +787,10 @@ module Sidekiq
708
787
  end
709
788
 
710
789
  ##
711
- # Allows enumeration of scheduled jobs within Sidekiq.
790
+ # The set of scheduled jobs within Sidekiq.
712
791
  # Based on this, you can search/filter for jobs. Here's an
713
- # example where I'm selecting all jobs of a certain type
714
- # and deleting them from the schedule queue.
792
+ # example where I'm selecting jobs based on some complex logic
793
+ # and deleting them from the scheduled set.
715
794
  #
716
795
  # r = Sidekiq::ScheduledSet.new
717
796
  # r.select do |scheduled|
@@ -726,7 +805,7 @@ module Sidekiq
726
805
  end
727
806
 
728
807
  ##
729
- # Allows enumeration of retries within Sidekiq.
808
+ # The set of retries within Sidekiq.
730
809
  # Based on this, you can search/filter for jobs. Here's an
731
810
  # example where I'm selecting all jobs of a certain type
732
811
  # and deleting them from the retry queue.
@@ -742,23 +821,29 @@ module Sidekiq
742
821
  super "retry"
743
822
  end
744
823
 
824
+ # Enqueues all jobs pending within the retry set.
745
825
  def retry_all
746
826
  each(&:retry) while size > 0
747
827
  end
748
828
 
829
+ # Kills all jobs pending within the retry set.
749
830
  def kill_all
750
831
  each(&:kill) while size > 0
751
832
  end
752
833
  end
753
834
 
754
835
  ##
755
- # Allows enumeration of dead jobs within Sidekiq.
836
+ # The set of dead jobs within Sidekiq. Dead jobs have failed all of
837
+ # their retries and are helding in this set pending some sort of manual
838
+ # fix. They will be removed after 6 months (dead_timeout) if not.
756
839
  #
757
840
  class DeadSet < JobSet
758
841
  def initialize
759
842
  super "dead"
760
843
  end
761
844
 
845
+ # Add the given job to the Dead set.
846
+ # @param message [String] the job data as JSON
762
847
  def kill(message, opts = {})
763
848
  now = Time.now.to_f
764
849
  Sidekiq.redis do |conn|
@@ -780,14 +865,19 @@ module Sidekiq
780
865
  true
781
866
  end
782
867
 
868
+ # Enqueue all dead jobs
783
869
  def retry_all
784
870
  each(&:retry) while size > 0
785
871
  end
786
872
 
873
+ # The maximum size of the Dead set. Older entries will be trimmed
874
+ # to stay within this limit. Default value is 10,000.
787
875
  def self.max_jobs
788
876
  Sidekiq[:dead_max_jobs]
789
877
  end
790
878
 
879
+ # The time limit for entries within the Dead set. Older entries will be thrown away.
880
+ # Default value is six months.
791
881
  def self.timeout
792
882
  Sidekiq[:dead_timeout_in_seconds]
793
883
  end
@@ -798,21 +888,28 @@ module Sidekiq
798
888
  # right now. Each process sends a heartbeat to Redis every 5 seconds
799
889
  # so this set should be relatively accurate, barring network partitions.
800
890
  #
801
- # Yields a Sidekiq::Process.
891
+ # @yieldparam [Sidekiq::Process]
802
892
  #
803
893
  class ProcessSet
804
894
  include Enumerable
805
895
 
896
+ # :nodoc:
897
+ # @api private
806
898
  def initialize(clean_plz = true)
807
899
  cleanup if clean_plz
808
900
  end
809
901
 
810
902
  # Cleans up dead processes recorded in Redis.
811
903
  # Returns the number of processes cleaned.
904
+ # :nodoc:
905
+ # @api private
812
906
  def cleanup
907
+ # dont run cleanup more than once per minute
908
+ return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
909
+
813
910
  count = 0
814
911
  Sidekiq.redis do |conn|
815
- procs = conn.sscan_each("processes").to_a.sort
912
+ procs = conn.sscan_each("processes").to_a
816
913
  heartbeats = conn.pipelined { |pipeline|
817
914
  procs.each do |key|
818
915
  pipeline.hget(key, "info")
@@ -863,6 +960,7 @@ module Sidekiq
863
960
  # based on current heartbeat. #each does that and ensures the set only
864
961
  # contains Sidekiq processes which have sent a heartbeat within the last
865
962
  # 60 seconds.
963
+ # @return [Integer] current number of registered Sidekiq processes
866
964
  def size
867
965
  Sidekiq.redis { |conn| conn.scard("processes") }
868
966
  end
@@ -870,10 +968,12 @@ module Sidekiq
870
968
  # Total number of threads available to execute jobs.
871
969
  # For Sidekiq Enterprise customers this number (in production) must be
872
970
  # less than or equal to your licensed concurrency.
971
+ # @return [Integer] the sum of process concurrency
873
972
  def total_concurrency
874
973
  sum { |x| x["concurrency"].to_i }
875
974
  end
876
975
 
976
+ # @return [Integer] total amount of RSS memory consumed by Sidekiq processes
877
977
  def total_rss_in_kb
878
978
  sum { |x| x["rss"].to_i }
879
979
  end
@@ -882,6 +982,8 @@ module Sidekiq
882
982
  # Returns the identity of the current cluster leader or "" if no leader.
883
983
  # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
884
984
  # or Sidekiq Pro.
985
+ # @return [String] Identity of cluster leader
986
+ # @return [String] empty string if no leader
885
987
  def leader
886
988
  @leader ||= begin
887
989
  x = Sidekiq.redis { |c| c.get("dear-leader") }
@@ -908,6 +1010,8 @@ module Sidekiq
908
1010
  # 'identity' => <unique string identifying the process>,
909
1011
  # }
910
1012
  class Process
1013
+ # :nodoc:
1014
+ # @api private
911
1015
  def initialize(hash)
912
1016
  @attribs = hash
913
1017
  end
@@ -932,18 +1036,31 @@ module Sidekiq
932
1036
  self["queues"]
933
1037
  end
934
1038
 
1039
+ # Signal this process to stop processing new jobs.
1040
+ # It will continue to execute jobs it has already fetched.
1041
+ # This method is *asynchronous* and it can take 5-10
1042
+ # seconds for the process to quiet.
935
1043
  def quiet!
936
1044
  signal("TSTP")
937
1045
  end
938
1046
 
1047
+ # Signal this process to shutdown.
1048
+ # It will shutdown within its configured :timeout value, default 25 seconds.
1049
+ # This method is *asynchronous* and it can take 5-10
1050
+ # seconds for the process to start shutting down.
939
1051
  def stop!
940
1052
  signal("TERM")
941
1053
  end
942
1054
 
1055
+ # Signal this process to log backtraces for all threads.
1056
+ # Useful if you have a frozen or deadlocked process which is
1057
+ # still sending a heartbeat.
1058
+ # This method is *asynchronous* and it can take 5-10 seconds.
943
1059
  def dump_threads
944
1060
  signal("TTIN")
945
1061
  end
946
1062
 
1063
+ # @return [Boolean] true if this process is quiet or shutting down
947
1064
  def stopping?
948
1065
  self["quiet"] == "true"
949
1066
  end
@@ -986,24 +1103,31 @@ module Sidekiq
986
1103
 
987
1104
  def each(&block)
988
1105
  results = []
1106
+ procs = nil
1107
+ all_works = nil
1108
+
989
1109
  Sidekiq.redis do |conn|
990
- procs = conn.sscan_each("processes").to_a
991
- procs.sort.each do |key|
992
- valid, workers = conn.pipelined { |pipeline|
993
- pipeline.exists?(key)
1110
+ procs = conn.sscan_each("processes").to_a.sort
1111
+
1112
+ all_works = conn.pipelined do |pipeline|
1113
+ procs.each do |key|
994
1114
  pipeline.hgetall("#{key}:work")
995
- }
996
- next unless valid
997
- workers.each_pair do |tid, json|
998
- hsh = Sidekiq.load_json(json)
999
- p = hsh["payload"]
1000
- # avoid breaking API, this is a side effect of the JSON optimization in #4316
1001
- hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
1002
- results << [key, tid, hsh]
1003
1115
  end
1004
1116
  end
1005
1117
  end
1006
1118
 
1119
+ procs.zip(all_works).each do |key, workers|
1120
+ workers.each_pair do |tid, json|
1121
+ next if json.empty?
1122
+
1123
+ hsh = Sidekiq.load_json(json)
1124
+ p = hsh["payload"]
1125
+ # avoid breaking API, this is a side effect of the JSON optimization in #4316
1126
+ hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
1127
+ results << [key, tid, hsh]
1128
+ end
1129
+ end
1130
+
1007
1131
  results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
1008
1132
  end
1009
1133
 
data/lib/sidekiq/cli.rb CHANGED
@@ -12,6 +12,17 @@ require "sidekiq"
12
12
  require "sidekiq/component"
13
13
  require "sidekiq/launcher"
14
14
 
15
+ # module ScoutApm
16
+ # VERSION = "5.3.1"
17
+ # end
18
+ fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
+
20
+
21
+ scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
+
23
+
24
+ EOM
25
+
15
26
  module Sidekiq # :nodoc:
16
27
  class CLI
17
28
  include Sidekiq::Component
@@ -214,6 +225,7 @@ module Sidekiq # :nodoc:
214
225
  # Both Sinatra 2.0+ and Sidekiq support this term.
215
226
  # RAILS_ENV and RACK_ENV are there for legacy support.
216
227
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
+ config[:environment] = @environment
217
229
  end
218
230
 
219
231
  def symbolize_keys_deep!(hash)
@@ -426,3 +438,4 @@ module Sidekiq # :nodoc:
426
438
  end
427
439
 
428
440
  require "sidekiq/systemd"
441
+ require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
@@ -176,7 +176,7 @@ module Sidekiq
176
176
  def enqueue_to_in(queue, interval, klass, *args)
177
177
  int = interval.to_f
178
178
  now = Time.now.to_f
179
- ts = (int < 1_000_000_000 ? now + int : int)
179
+ ts = ((int < 1_000_000_000) ? now + int : int)
180
180
 
181
181
  item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
182
182
  item.delete("at") if ts <= now
@@ -231,7 +231,7 @@ module Sidekiq
231
231
  entry["enqueued_at"] = now
232
232
  Sidekiq.dump_json(entry)
233
233
  }
234
- conn.sadd("queues", queue)
234
+ conn.sadd("queues", [queue])
235
235
  conn.lpush("queue:#{queue}", to_push)
236
236
  end
237
237
  end
@@ -47,6 +47,7 @@ module Sidekiq
47
47
  end
48
48
 
49
49
  def fire_event(event, options = {})
50
+ oneshot = options.fetch(:oneshot, true)
50
51
  reverse = options[:reverse]
51
52
  reraise = options[:reraise]
52
53
 
@@ -58,7 +59,7 @@ module Sidekiq
58
59
  handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
59
60
  raise ex if reraise
60
61
  end
61
- arr.clear # once we've fired an event, we never fire it again
62
+ arr.clear if oneshot # once we've fired an event, we never fire it again
62
63
  end
63
64
  end
64
65
  end