sidekiq 7.2.4 → 7.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +116 -0
  3. data/README.md +1 -1
  4. data/bin/sidekiqload +21 -12
  5. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +75 -0
  6. data/lib/generators/sidekiq/job_generator.rb +2 -0
  7. data/lib/sidekiq/api.rb +63 -34
  8. data/lib/sidekiq/capsule.rb +8 -3
  9. data/lib/sidekiq/cli.rb +2 -1
  10. data/lib/sidekiq/client.rb +21 -1
  11. data/lib/sidekiq/component.rb +22 -0
  12. data/lib/sidekiq/config.rb +27 -3
  13. data/lib/sidekiq/deploy.rb +2 -0
  14. data/lib/sidekiq/embedded.rb +2 -0
  15. data/lib/sidekiq/fetch.rb +1 -1
  16. data/lib/sidekiq/iterable_job.rb +55 -0
  17. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  18. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  19. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  20. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  21. data/lib/sidekiq/job/iterable.rb +294 -0
  22. data/lib/sidekiq/job.rb +13 -2
  23. data/lib/sidekiq/job_logger.rb +7 -6
  24. data/lib/sidekiq/job_retry.rb +6 -1
  25. data/lib/sidekiq/job_util.rb +2 -0
  26. data/lib/sidekiq/launcher.rb +1 -1
  27. data/lib/sidekiq/metrics/query.rb +2 -0
  28. data/lib/sidekiq/metrics/shared.rb +15 -4
  29. data/lib/sidekiq/metrics/tracking.rb +13 -5
  30. data/lib/sidekiq/middleware/current_attributes.rb +46 -13
  31. data/lib/sidekiq/middleware/modules.rb +2 -0
  32. data/lib/sidekiq/monitor.rb +2 -1
  33. data/lib/sidekiq/paginator.rb +6 -0
  34. data/lib/sidekiq/processor.rb +20 -10
  35. data/lib/sidekiq/rails.rb +12 -0
  36. data/lib/sidekiq/redis_client_adapter.rb +8 -5
  37. data/lib/sidekiq/redis_connection.rb +33 -2
  38. data/lib/sidekiq/ring_buffer.rb +2 -0
  39. data/lib/sidekiq/systemd.rb +2 -0
  40. data/lib/sidekiq/testing.rb +5 -5
  41. data/lib/sidekiq/version.rb +5 -1
  42. data/lib/sidekiq/web/action.rb +21 -4
  43. data/lib/sidekiq/web/application.rb +43 -82
  44. data/lib/sidekiq/web/helpers.rb +62 -15
  45. data/lib/sidekiq/web/router.rb +5 -2
  46. data/lib/sidekiq/web.rb +54 -2
  47. data/lib/sidekiq.rb +5 -3
  48. data/sidekiq.gemspec +3 -2
  49. data/web/assets/javascripts/application.js +6 -1
  50. data/web/assets/javascripts/dashboard-charts.js +24 -12
  51. data/web/assets/javascripts/dashboard.js +7 -1
  52. data/web/assets/stylesheets/application.css +16 -3
  53. data/web/locales/en.yml +3 -1
  54. data/web/locales/fr.yml +0 -1
  55. data/web/locales/gd.yml +0 -1
  56. data/web/locales/it.yml +32 -1
  57. data/web/locales/ja.yml +0 -1
  58. data/web/locales/pt-br.yml +1 -2
  59. data/web/locales/tr.yml +100 -0
  60. data/web/locales/uk.yml +24 -1
  61. data/web/locales/zh-cn.yml +0 -1
  62. data/web/locales/zh-tw.yml +0 -1
  63. data/web/views/_footer.erb +1 -2
  64. data/web/views/dashboard.erb +10 -7
  65. data/web/views/filtering.erb +1 -2
  66. data/web/views/layout.erb +6 -6
  67. data/web/views/metrics.erb +7 -8
  68. data/web/views/metrics_for_job.erb +4 -4
  69. data/web/views/morgue.erb +2 -2
  70. data/web/views/queue.erb +1 -1
  71. metadata +32 -13
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6c43e6b585c25dcfc8ef8364bb36cf74f9167b981ad03faa3a8d76e0d45ebe55
4
- data.tar.gz: d8c65dc03008f7280b36af94db753d4c7f68267c2eb0d78cd018322887aabbb0
3
+ metadata.gz: dccc402e8f4e309a47a2adb62418fa29989012e8c94f34aa11200b098dc375f2
4
+ data.tar.gz: 71b64a582d5971b045d60bae38dc26da265b60ff9a099ca3b2f9c7df9035e805
5
5
  SHA512:
6
- metadata.gz: d2687692b873ab82bda2ad32e9be795150cd0a8d3d330bc19f5b509ba729bef33189e06ebac86b1906c2682187391d6cf0d532e47d03fcbea83058109c5816ef
7
- data.tar.gz: 431a482baeb03fc4de50fbdfba8717fc332a9d6564fde98a77699a7bd174fa3194431385951cf689c64e04853039c95fcf287084f283e8d381b3b37d5bc665e0
6
+ metadata.gz: d9148b613a222ca9617ceebe25bb04a5d086edacdafad4a426b6232662d3b583db462dc5e0491297f7859037c5166bcb541bddcd3949f6d3b5a1c2e3fc572b65
7
+ data.tar.gz: 93a797cefd1a68adb236c7538c28571467c328d6965af0bc5ce505a1391d6f5b6e2ae2c87b42110e837dcfa8d788aff55309806bbb1bcd181e205c3bc66adc29
data/Changes.md CHANGED
@@ -2,6 +2,122 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
+ 7.3.9
6
+ ----------
7
+
8
+ - Only require activejob if necessary [#6584]
9
+ - Fix iterable job cancellation [#6589]
10
+ - Web UI accessibility improvements [#6604]
11
+
12
+ 7.3.8
13
+ ----------
14
+
15
+ - Fix dead tag links [#6554]
16
+ - Massive Web UI performance improvement, some pages up to 15x faster [#6555]
17
+
18
+ 7.3.7
19
+ ----------
20
+
21
+ - Backport `Sidekiq::Web.configure` for compatibility with 8.0 [#6532]
22
+ - Backport `url_params(key)` and `route_params(key)` for compatibility with 8.0 [#6532]
23
+ - Various fixes for UI filtering [#6508]
24
+ - Tune `inspect` for internal S::Components to keep size managable [#6553]
25
+
26
+ 7.3.6
27
+ ----------
28
+
29
+ - Forward compatibility fixes for Ruby 3.4
30
+ - Filtering in the Web UI now works via GET so you can bookmark a filtered view. [#6497]
31
+
32
+ 7.3.5
33
+ ----------
34
+
35
+ - Reimplement `retry_all` and `kill_all` API methods to use ZPOPMIN,
36
+ approximately 30-60% faster. [#6481]
37
+ - Add preload testing binary at `examples/testing/sidekiq_boot` to verify your Rails app boots correctly with Sidekiq Enterprise's app preloading.
38
+ - Fix circular require with ActiveJob adapter [#6477]
39
+ - Fix potential race condition leading to incorrect serialized values for CurrentAttributes [#6475]
40
+ - Restore missing elapsed time when default job logging is disabled
41
+
42
+ 7.3.4
43
+ ----------
44
+
45
+ - Fix FrozenError when starting Sidekiq [#6470]
46
+
47
+ 7.3.3
48
+ ----------
49
+
50
+ - Freeze global configuration once boot is complete, to avoid configuration race conditions [#6466, #6465]
51
+ - Sidekiq now warns if a job iteration takes longer than the `-t` timeout setting (defaults to 25 seconds)
52
+ - Iteration callbacks now have easy access to job arguments via the `arguments` method:
53
+ ```ruby
54
+ def on_stop
55
+ p arguments # => `[123, "string", {"key" => "value"}]`
56
+ id, str, hash = arguments
57
+ end
58
+ ```
59
+ - Iterable jobs can be cancelled via `Sidekiq::Client#cancel!`:
60
+ ```ruby
61
+ c = Sidekiq::Client.new
62
+ jid = c.push("class" => SomeJob, "args" => [123])
63
+ c.cancel!(jid) # => true
64
+ ```
65
+ - Take over support for ActiveJob's `:sidekiq` adapter [#6430, fatkodima]
66
+ - Ensure CurrentAttributes are in scope when creating batch callbacks [#6455]
67
+ - Add `Sidekiq.gem_version` API.
68
+ - Update Ukranian translations
69
+
70
+ 7.3.2
71
+ ----------
72
+
73
+ - Adjust ActiveRecord batch iteration to restart an interrupted batch from the beginning.
74
+ Each batch should be processed as a single transaction in order to be idempotent. [#6405]
75
+ - Fix typo in Sidekiq::DeadSet#kill [#6397]
76
+ - Fix CSS issue with bottom bar in Web UI [#6414]
77
+
78
+ 7.3.1
79
+ ----------
80
+
81
+ - Don't count job interruptions as failures in metrics [#6386]
82
+ - Add frozen string literal to a number of .rb files.
83
+ - Fix frozen string error with style_tag and script_tag [#6371]
84
+ - Fix an error on Ruby 2.7 because of usage of `Hash#except` [#6376]
85
+
86
+ 7.3.0
87
+ ----------
88
+
89
+ - **NEW FEATURE** Add `Sidekiq::IterableJob`, iteration support for long-running jobs. [#6286, fatkodima]
90
+ Iterable jobs are interruptible and can restart quickly if
91
+ running during a deploy. You must ensure that `each_iteration`
92
+ doesn't take more than Sidekiq's `-t` timeout (default: 25 seconds). Iterable jobs must not implement `perform`.
93
+ ```ruby
94
+ class ProcessArrayJob
95
+ include Sidekiq::IterableJob
96
+ def build_enumerator(*args, **kwargs)
97
+ array_enumerator(args, **kwargs)
98
+ end
99
+ def each_iteration(arg)
100
+ puts arg
101
+ end
102
+ end
103
+ ProcessArrayJob.perform_async(1, 2, 3)
104
+ ```
105
+ See the [Iteration](//github.com/sidekiq/sidekiq/wiki/Iteration) wiki page and the RDoc in `Sidekiq::IterableJob`.
106
+ This feature should be considered BETA until the next minor release.
107
+ - **SECURITY** The Web UI no longer allows extensions to use `<script>`.
108
+ Adjust CSP to disallow inline scripts within the Web UI. Please see
109
+ `examples/webui-ext` for how to register Web UI extensions and use
110
+ dynamic CSS and JS. This will make Sidekiq immune to XSS attacks. [#6270]
111
+ - Add config option, `:skip_default_job_logging` to disable Sidekiq's default
112
+ start/finish job logging. [#6200]
113
+ - Allow `Sidekiq::Limiter.redis` to use Redis Cluster [#6288]
114
+ - Retain CurrentAttributeѕ after inline execution [#6307]
115
+ - Ignore non-existent CurrentAttributes attributes when restoring [#6341]
116
+ - Raise default Redis {read,write,connect} timeouts from 1 to 3 seconds
117
+ to minimize ReadTimeoutErrors [#6162]
118
+ - Add `logger` as a dependency since it will become bundled in Ruby 3.5 [#6320]
119
+ - Ignore unsupported locales in the Web UI [#6313]
120
+
5
121
  7.2.4
6
122
  ----------
7
123
 
data/README.md CHANGED
@@ -86,7 +86,7 @@ Useful resources:
86
86
  * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
87
87
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
88
88
 
89
- Every Friday morning is Sidekiq office hour: I video chat and answer questions.
89
+ Every Thursday morning is Sidekiq office hour: I video chat and answer questions.
90
90
  See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
91
91
 
92
92
  Contributing
data/bin/sidekiqload CHANGED
@@ -50,7 +50,7 @@ if ENV["AJ"]
50
50
  ActiveJob::Base.logger.level = Logger::WARN
51
51
 
52
52
  class LoadJob < ActiveJob::Base
53
- def perform(idx, ts = nil)
53
+ def perform(string, idx, hash, ts = nil)
54
54
  puts(Time.now.to_f - ts) if !ts.nil?
55
55
  end
56
56
  end
@@ -58,12 +58,21 @@ end
58
58
 
59
59
  class LoadWorker
60
60
  include Sidekiq::Job
61
+ $count = 0
62
+ $lock = Mutex.new
63
+
61
64
  sidekiq_options retry: 1
62
65
  sidekiq_retry_in do |x|
63
66
  1
64
67
  end
65
68
 
66
- def perform(idx, ts = nil)
69
+ def perform(string, idx, hash, ts = nil)
70
+ $lock.synchronize do
71
+ $count += 1
72
+ if $count % 100_000 == 0
73
+ logger.warn("#{Time.now} Done #{$count}")
74
+ end
75
+ end
67
76
  puts(Time.now.to_f - ts) if !ts.nil?
68
77
  # raise idx.to_s if idx % 100 == 1
69
78
  end
@@ -133,13 +142,13 @@ class Loader
133
142
  start = Time.now
134
143
  if ENV["AJ"]
135
144
  @iter.times do
136
- @count.times do |idx|
137
- LoadJob.perform_later(idx)
138
- end
145
+ ActiveJob.perform_all_later(@count.times.map do |idx|
146
+ LoadJob.new("mike", idx, {mike: "bob"})
147
+ end)
139
148
  end
140
149
  else
141
150
  @iter.times do
142
- arr = Array.new(@count) { |idx| [idx] }
151
+ arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
143
152
  Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
144
153
  end
145
154
  end
@@ -163,13 +172,13 @@ class Loader
163
172
  Sidekiq.logger.error("Now here's the latency for three jobs")
164
173
 
165
174
  if ENV["AJ"]
166
- LoadJob.perform_later(1, Time.now.to_f)
167
- LoadJob.perform_later(2, Time.now.to_f)
168
- LoadJob.perform_later(3, Time.now.to_f)
175
+ LoadJob.perform_later("", 1, {}, Time.now.to_f)
176
+ LoadJob.perform_later("", 2, {}, Time.now.to_f)
177
+ LoadJob.perform_later("", 3, {}, Time.now.to_f)
169
178
  else
170
- LoadWorker.perform_async(1, Time.now.to_f)
171
- LoadWorker.perform_async(2, Time.now.to_f)
172
- LoadWorker.perform_async(3, Time.now.to_f)
179
+ LoadWorker.perform_async("", 1, {}, Time.now.to_f)
180
+ LoadWorker.perform_async("", 2, {}, Time.now.to_f)
181
+ LoadWorker.perform_async("", 3, {}, Time.now.to_f)
173
182
  end
174
183
 
175
184
  sleep 0.1
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveJob
4
+ module QueueAdapters
5
+ # Explicitly remove the implementation existing in older rails'.
6
+ remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
7
+
8
+ # Sidekiq adapter for Active Job
9
+ #
10
+ # To use Sidekiq set the queue_adapter config to +:sidekiq+.
11
+ #
12
+ # Rails.application.config.active_job.queue_adapter = :sidekiq
13
+ class SidekiqAdapter
14
+ # Defines whether enqueuing should happen implicitly to after commit when called
15
+ # from inside a transaction.
16
+ # @api private
17
+ def enqueue_after_transaction_commit?
18
+ true
19
+ end
20
+
21
+ # @api private
22
+ def enqueue(job)
23
+ job.provider_job_id = JobWrapper.set(
24
+ wrapped: job.class,
25
+ queue: job.queue_name
26
+ ).perform_async(job.serialize)
27
+ end
28
+
29
+ # @api private
30
+ def enqueue_at(job, timestamp)
31
+ job.provider_job_id = JobWrapper.set(
32
+ wrapped: job.class,
33
+ queue: job.queue_name
34
+ ).perform_at(timestamp, job.serialize)
35
+ end
36
+
37
+ # @api private
38
+ def enqueue_all(jobs)
39
+ enqueued_count = 0
40
+ jobs.group_by(&:class).each do |job_class, same_class_jobs|
41
+ same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
42
+ immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
43
+
44
+ if immediate_jobs.any?
45
+ jids = Sidekiq::Client.push_bulk(
46
+ "class" => JobWrapper,
47
+ "wrapped" => job_class,
48
+ "queue" => queue,
49
+ "args" => immediate_jobs.map { |job| [job.serialize] }
50
+ )
51
+ enqueued_count += jids.compact.size
52
+ end
53
+
54
+ if scheduled_jobs.any?
55
+ jids = Sidekiq::Client.push_bulk(
56
+ "class" => JobWrapper,
57
+ "wrapped" => job_class,
58
+ "queue" => queue,
59
+ "args" => scheduled_jobs.map { |job| [job.serialize] },
60
+ "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
61
+ )
62
+ enqueued_count += jids.compact.size
63
+ end
64
+ end
65
+ end
66
+ enqueued_count
67
+ end
68
+
69
+ # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
70
+ # @api private
71
+ class JobWrapper < Sidekiq::ActiveJob::Wrapper
72
+ end
73
+ end
74
+ end
75
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "rails/generators/named_base"
2
4
 
3
5
  module Sidekiq
data/lib/sidekiq/api.rb CHANGED
@@ -373,7 +373,7 @@ module Sidekiq
373
373
  def display_class
374
374
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
375
375
  @klass ||= self["display_class"] || begin
376
- if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
376
+ if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
377
377
  job_class = @item["wrapped"] || args[0]
378
378
  if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
379
379
  # MailerClass#mailer_method
@@ -389,7 +389,7 @@ module Sidekiq
389
389
 
390
390
  def display_args
391
391
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
392
- @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
392
+ @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
393
393
  job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
394
394
  if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
395
395
  # remove MailerClass, mailer_method and 'deliver_now'
@@ -668,6 +668,41 @@ module Sidekiq
668
668
  end
669
669
  end
670
670
 
671
+ def pop_each
672
+ Sidekiq.redis do |c|
673
+ size.times do
674
+ data, score = c.zpopmin(name, 1)&.first
675
+ break unless data
676
+ yield data, score
677
+ end
678
+ end
679
+ end
680
+
681
+ def retry_all
682
+ c = Sidekiq::Client.new
683
+ pop_each do |msg, _|
684
+ job = Sidekiq.load_json(msg)
685
+ # Manual retries should not count against the retry limit.
686
+ job["retry_count"] -= 1 if job["retry_count"]
687
+ c.push(job)
688
+ end
689
+ end
690
+
691
+ # Move all jobs from this Set to the Dead Set.
692
+ # See DeadSet#kill
693
+ def kill_all(notify_failure: false, ex: nil)
694
+ ds = DeadSet.new
695
+ opts = {notify_failure: notify_failure, ex: ex, trim: false}
696
+
697
+ begin
698
+ pop_each do |msg, _|
699
+ ds.kill(msg, opts)
700
+ end
701
+ ensure
702
+ ds.trim
703
+ end
704
+ end
705
+
671
706
  def each
672
707
  initial_size = @_size
673
708
  offset_size = 0
@@ -765,10 +800,6 @@ module Sidekiq
765
800
 
766
801
  ##
767
802
  # The set of scheduled jobs within Sidekiq.
768
- # Based on this, you can search/filter for jobs. Here's an
769
- # example where I'm selecting jobs based on some complex logic
770
- # and deleting them from the scheduled set.
771
- #
772
803
  # See the API wiki page for usage notes and examples.
773
804
  #
774
805
  class ScheduledSet < JobSet
@@ -779,26 +810,12 @@ module Sidekiq
779
810
 
780
811
  ##
781
812
  # The set of retries within Sidekiq.
782
- # Based on this, you can search/filter for jobs. Here's an
783
- # example where I'm selecting all jobs of a certain type
784
- # and deleting them from the retry queue.
785
- #
786
813
  # See the API wiki page for usage notes and examples.
787
814
  #
788
815
  class RetrySet < JobSet
789
816
  def initialize
790
817
  super("retry")
791
818
  end
792
-
793
- # Enqueues all jobs pending within the retry set.
794
- def retry_all
795
- each(&:retry) while size > 0
796
- end
797
-
798
- # Kills all jobs pending within the retry set.
799
- def kill_all
800
- each(&:kill) while size > 0
801
- end
802
819
  end
803
820
 
804
821
  ##
@@ -811,33 +828,45 @@ module Sidekiq
811
828
  super("dead")
812
829
  end
813
830
 
831
+ # Trim dead jobs which are over our storage limits
832
+ def trim
833
+ hash = Sidekiq.default_configuration
834
+ now = Time.now.to_f
835
+ Sidekiq.redis do |conn|
836
+ conn.multi do |transaction|
837
+ transaction.zremrangebyscore(name, "-inf", now - hash[:dead_timeout_in_seconds])
838
+ transaction.zremrangebyrank(name, 0, - hash[:dead_max_jobs])
839
+ end
840
+ end
841
+ end
842
+
814
843
  # Add the given job to the Dead set.
815
844
  # @param message [String] the job data as JSON
845
+ # @option opts [Boolean] :notify_failure (true) Whether death handlers should be called
846
+ # @option opts [Boolean] :trim (true) Whether Sidekiq should trim the structure to keep it within configuration
847
+ # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
816
848
  def kill(message, opts = {})
817
849
  now = Time.now.to_f
818
850
  Sidekiq.redis do |conn|
819
- conn.multi do |transaction|
820
- transaction.zadd(name, now.to_s, message)
821
- transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
822
- transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
823
- end
851
+ conn.zadd(name, now.to_s, message)
824
852
  end
825
853
 
854
+ trim if opts[:trim] != false
855
+
826
856
  if opts[:notify_failure] != false
827
857
  job = Sidekiq.load_json(message)
828
- r = RuntimeError.new("Job killed by API")
829
- r.set_backtrace(caller)
858
+ if opts[:ex]
859
+ ex = opts[:ex]
860
+ else
861
+ ex = RuntimeError.new("Job killed by API")
862
+ ex.set_backtrace(caller)
863
+ end
830
864
  Sidekiq.default_configuration.death_handlers.each do |handle|
831
- handle.call(job, r)
865
+ handle.call(job, ex)
832
866
  end
833
867
  end
834
868
  true
835
869
  end
836
-
837
- # Enqueue all dead jobs
838
- def retry_all
839
- each(&:retry) while size > 0
840
- end
841
870
  end
842
871
 
843
872
  ##
@@ -1199,7 +1228,7 @@ module Sidekiq
1199
1228
  @hsh.send(*all)
1200
1229
  end
1201
1230
 
1202
- def respond_to_missing?(name)
1231
+ def respond_to_missing?(name, *args)
1203
1232
  @hsh.respond_to?(name)
1204
1233
  end
1205
1234
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "sidekiq/component"
2
4
 
3
5
  module Sidekiq
@@ -17,6 +19,7 @@ module Sidekiq
17
19
  # end
18
20
  class Capsule
19
21
  include Sidekiq::Component
22
+ extend Forwardable
20
23
 
21
24
  attr_reader :name
22
25
  attr_reader :queues
@@ -24,6 +27,8 @@ module Sidekiq
24
27
  attr_reader :mode
25
28
  attr_reader :weights
26
29
 
30
+ def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
31
+
27
32
  def initialize(name, config)
28
33
  @name = name
29
34
  @config = config
@@ -35,9 +40,9 @@ module Sidekiq
35
40
 
36
41
  def fetcher
37
42
  @fetcher ||= begin
38
- inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
39
- inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
40
- inst
43
+ instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
44
+ instance.setup(config[:fetch_setup]) if instance.respond_to?(:setup)
45
+ instance
41
46
  end
42
47
  end
43
48
 
data/lib/sidekiq/cli.rb CHANGED
@@ -101,7 +101,7 @@ module Sidekiq # :nodoc:
101
101
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
102
102
  @config.server_middleware
103
103
 
104
- ::Process.warmup if warmup && ::Process.respond_to?(:warmup)
104
+ ::Process.warmup if warmup && ::Process.respond_to?(:warmup) && ENV["RUBY_DISABLE_WARMUP"] != "1"
105
105
 
106
106
  # Before this point, the process is initializing with just the main thread.
107
107
  # Starting here the process will now have multiple threads running.
@@ -423,3 +423,4 @@ end
423
423
 
424
424
  require "sidekiq/systemd"
425
425
  require "sidekiq/metrics/tracking"
426
+ require "sidekiq/job/interrupt_handler"
@@ -58,6 +58,23 @@ module Sidekiq
58
58
  end
59
59
  end
60
60
 
61
+ # Cancel the IterableJob with the given JID.
62
+ # **NB: Cancellation is asynchronous.** Iteration checks every
63
+ # five seconds so this will not immediately stop the given job.
64
+ def cancel!(jid)
65
+ key = "it-#{jid}"
66
+ _, result, _ = Sidekiq.redis do |c|
67
+ c.pipelined do |p|
68
+ p.hsetnx(key, "cancelled", Time.now.to_i)
69
+ p.hget(key, "cancelled")
70
+ p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
71
+ # TODO When Redis 7.2 is required
72
+ # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
73
+ end
74
+ end
75
+ result.to_i
76
+ end
77
+
61
78
  ##
62
79
  # The main method used to push a job to Redis. Accepts a number of options:
63
80
  #
@@ -248,9 +265,12 @@ module Sidekiq
248
265
  def atomic_push(conn, payloads)
249
266
  if payloads.first.key?("at")
250
267
  conn.zadd("schedule", payloads.flat_map { |hash|
251
- at = hash.delete("at").to_s
268
+ at = hash["at"].to_s
252
269
  # ActiveJob sets this but the job has not been enqueued yet
253
270
  hash.delete("enqueued_at")
271
+ # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
272
+ hash = hash.dup
273
+ hash.delete("at")
254
274
  [at, Sidekiq.dump_json(hash)]
255
275
  })
256
276
  else
@@ -64,5 +64,27 @@ module Sidekiq
64
64
  end
65
65
  arr.clear if oneshot # once we've fired an event, we never fire it again
66
66
  end
67
+
68
+ # When you have a large tree of components, the `inspect` output
69
+ # can get out of hand, especially with lots of Sidekiq::Config
70
+ # references everywhere. We avoid calling `inspect` on more complex
71
+ # state and use `to_s` instead to keep output manageable, #6553
72
+ def inspect
73
+ "#<#{self.class.name} #{
74
+ instance_variables.map do |name|
75
+ value = instance_variable_get(name)
76
+ case value
77
+ when Proc
78
+ "#{name}=#{value}"
79
+ when Sidekiq::Config
80
+ "#{name}=#{value}"
81
+ when Sidekiq::Component
82
+ "#{name}=#{value}"
83
+ else
84
+ "#{name}=#{value.inspect}"
85
+ end
86
+ end.join(", ")
87
+ }>"
88
+ end
67
89
  end
68
90
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "forwardable"
2
4
 
3
5
  require "set"
@@ -17,6 +19,10 @@ module Sidekiq
17
19
  poll_interval_average: nil,
18
20
  average_scheduled_poll_interval: 5,
19
21
  on_complex_arguments: :raise,
22
+ iteration: {
23
+ max_job_runtime: nil,
24
+ retry_backoff: 0
25
+ },
20
26
  error_handlers: [],
21
27
  death_handlers: [],
22
28
  lifecycle_events: {
@@ -52,9 +58,15 @@ module Sidekiq
52
58
  @capsules = {}
53
59
  end
54
60
 
55
- def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
61
+ def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
56
62
  attr_reader :capsules
57
63
 
64
+ def inspect
65
+ "#<#{self.class.name} @options=#{
66
+ @options.except(:lifecycle_events, :reloader, :death_handlers, :error_handlers).inspect
67
+ }>"
68
+ end
69
+
58
70
  def to_json(*)
59
71
  Sidekiq.dump_json(@options)
60
72
  end
@@ -179,7 +191,13 @@ module Sidekiq
179
191
 
180
192
  # register global singletons which can be accessed elsewhere
181
193
  def register(name, instance)
182
- @directory[name] = instance
194
+ # logger.debug("register[#{name}] = #{instance}")
195
+ # Sidekiq Enterprise lazy registers a few services so we
196
+ # can't lock down this hash completely.
197
+ hash = @directory.dup
198
+ hash[name] = instance
199
+ @directory = hash.freeze
200
+ instance
183
201
  end
184
202
 
185
203
  # find a singleton
@@ -187,10 +205,16 @@ module Sidekiq
187
205
  # JNDI is just a fancy name for a hash lookup
188
206
  @directory.fetch(name) do |key|
189
207
  return nil unless default_class
190
- @directory[key] = default_class.new(self)
208
+ register(key, default_class.new(self))
191
209
  end
192
210
  end
193
211
 
212
+ def freeze!
213
+ @directory.freeze
214
+ @options.freeze
215
+ true
216
+ end
217
+
194
218
  ##
195
219
  # Death handlers are called when all retries for a job have been exhausted and
196
220
  # the job dies. It's the notification to your application
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "sidekiq/redis_connection"
2
4
  require "time"
3
5
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "sidekiq/component"
2
4
  require "sidekiq/launcher"
3
5
  require "sidekiq/metrics/tracking"
data/lib/sidekiq/fetch.rb CHANGED
@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
44
44
  return nil
45
45
  end
46
46
 
47
- queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
47
+ queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
48
48
  UnitOfWork.new(queue, job, config) if queue
49
49
  end
50
50