sidekiq 6.5.12 → 7.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +5 -24
  3. data/README.md +13 -12
  4. data/bin/sidekiq +3 -8
  5. data/bin/sidekiqload +14 -23
  6. data/lib/sidekiq/api.rb +51 -127
  7. data/lib/sidekiq/capsule.rb +110 -0
  8. data/lib/sidekiq/cli.rb +44 -59
  9. data/lib/sidekiq/client.rb +30 -17
  10. data/lib/sidekiq/component.rb +1 -0
  11. data/lib/sidekiq/config.rb +270 -0
  12. data/lib/sidekiq/deploy.rb +62 -0
  13. data/lib/sidekiq/embedded.rb +61 -0
  14. data/lib/sidekiq/fetch.rb +10 -11
  15. data/lib/sidekiq/job.rb +375 -10
  16. data/lib/sidekiq/job_logger.rb +1 -1
  17. data/lib/sidekiq/job_retry.rb +8 -8
  18. data/lib/sidekiq/job_util.rb +4 -4
  19. data/lib/sidekiq/launcher.rb +36 -46
  20. data/lib/sidekiq/logger.rb +1 -26
  21. data/lib/sidekiq/manager.rb +9 -11
  22. data/lib/sidekiq/metrics/query.rb +3 -3
  23. data/lib/sidekiq/metrics/shared.rb +4 -3
  24. data/lib/sidekiq/metrics/tracking.rb +18 -18
  25. data/lib/sidekiq/middleware/chain.rb +7 -9
  26. data/lib/sidekiq/middleware/current_attributes.rb +3 -8
  27. data/lib/sidekiq/monitor.rb +1 -1
  28. data/lib/sidekiq/paginator.rb +1 -9
  29. data/lib/sidekiq/pool.rb +7 -0
  30. data/lib/sidekiq/processor.rb +17 -26
  31. data/lib/sidekiq/rails.rb +11 -10
  32. data/lib/sidekiq/redis_client_adapter.rb +9 -45
  33. data/lib/sidekiq/redis_connection.rb +11 -111
  34. data/lib/sidekiq/scheduled.rb +19 -20
  35. data/lib/sidekiq/testing.rb +4 -32
  36. data/lib/sidekiq/transaction_aware_client.rb +4 -5
  37. data/lib/sidekiq/version.rb +1 -1
  38. data/lib/sidekiq/web/application.rb +1 -4
  39. data/lib/sidekiq/web/csrf_protection.rb +1 -1
  40. data/lib/sidekiq/web/helpers.rb +21 -22
  41. data/lib/sidekiq/web.rb +2 -17
  42. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  43. data/lib/sidekiq.rb +52 -274
  44. data/sidekiq.gemspec +29 -5
  45. data/web/assets/javascripts/application.js +0 -1
  46. data/web/assets/javascripts/base-charts.js +106 -0
  47. data/web/assets/javascripts/dashboard-charts.js +166 -0
  48. data/web/assets/javascripts/dashboard.js +3 -223
  49. data/web/assets/javascripts/metrics.js +90 -116
  50. data/web/assets/stylesheets/application-rtl.css +2 -91
  51. data/web/assets/stylesheets/application.css +21 -296
  52. data/web/locales/ar.yml +70 -70
  53. data/web/locales/cs.yml +62 -62
  54. data/web/locales/da.yml +52 -52
  55. data/web/locales/de.yml +65 -65
  56. data/web/locales/el.yml +2 -7
  57. data/web/locales/en.yml +76 -70
  58. data/web/locales/es.yml +68 -68
  59. data/web/locales/fa.yml +65 -65
  60. data/web/locales/fr.yml +67 -67
  61. data/web/locales/he.yml +65 -64
  62. data/web/locales/hi.yml +59 -59
  63. data/web/locales/it.yml +53 -53
  64. data/web/locales/ja.yml +64 -68
  65. data/web/locales/ko.yml +52 -52
  66. data/web/locales/lt.yml +66 -66
  67. data/web/locales/nb.yml +61 -61
  68. data/web/locales/nl.yml +52 -52
  69. data/web/locales/pl.yml +45 -45
  70. data/web/locales/pt-br.yml +59 -69
  71. data/web/locales/pt.yml +51 -51
  72. data/web/locales/ru.yml +67 -66
  73. data/web/locales/sv.yml +53 -53
  74. data/web/locales/ta.yml +60 -60
  75. data/web/locales/uk.yml +62 -61
  76. data/web/locales/ur.yml +64 -64
  77. data/web/locales/vi.yml +67 -67
  78. data/web/locales/zh-cn.yml +1 -0
  79. data/web/locales/zh-tw.yml +10 -1
  80. data/web/views/_footer.erb +5 -2
  81. data/web/views/busy.erb +1 -6
  82. data/web/views/dashboard.erb +36 -5
  83. data/web/views/metrics.erb +30 -19
  84. data/web/views/metrics_for_job.erb +16 -34
  85. metadata +60 -37
  86. data/lib/sidekiq/delay.rb +0 -43
  87. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  88. data/lib/sidekiq/extensions/active_record.rb +0 -43
  89. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  90. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  91. data/lib/sidekiq/metrics/deploy.rb +0 -47
  92. data/lib/sidekiq/worker.rb +0 -370
  93. data/web/assets/javascripts/graph.js +0 -16
  94. /data/{LICENSE → LICENSE.txt} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 55c6f9a8c0f2810bcbe7744c95204893d73f534666f6091c74dcb5199e7a4a28
4
- data.tar.gz: c42690ef0d1876c94eb51fb68319275d11f10169826180db921b34d6334a1a54
3
+ metadata.gz: 31722cec9da8b32488242bbcf40b46f3e2eaf8f9e5f5acefb4be306e6a3521e1
4
+ data.tar.gz: 625639dd35227dd5c9e36d04e93afbb3f8ae6372c532625453a0d681fac19a65
5
5
  SHA512:
6
- metadata.gz: 461b559e18cbdf8fe6ba20ab9fa38d08fd3b7b8d14dce7a7f9369d678e92bd25c07734bc14b0167f83589c0f03501897195b3fdf9cfd5de5a60f039bf7436f98
7
- data.tar.gz: ce0490b438a22a71c37e5a65193a7728e3297b437730d30a3653807ff1e89476db1f8c62d8de50c0c8cebbbce2ba6fb65f110855e071c62746fe8697a0f636e1
6
+ metadata.gz: 9471e077c2122b7b8fe94d075d1203d83d463608485852237ede31f9307b57233bb6cb0ba867f8556c080d61b01a5eb5efb7f1e3d68835e5bd16adc2630c65e1
7
+ data.tar.gz: 95b6c47592ba8641b4398859dca9c80b0d644c9e61cffec5482ff5741047daa9523f9cc5efb1974e197f3f06cab07321a5aca8b1838635372b3ea26e37cfca45
data/Changes.md CHANGED
@@ -2,28 +2,9 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
- 6.5.11
5
+ HEAD
6
6
  ----------
7
7
 
8
- - Fix for Rails 7.1 [#6067]
9
-
10
- 6.5.10
11
- ----------
12
-
13
- - Web UI DoS vector [#6045] CVE-2023-26141
14
- - Fix broadcast logger with Rails 7.1 [#6054]
15
-
16
- 6.5.9
17
- ----------
18
-
19
- - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
20
-
21
- 6.5.8
22
- ----------
23
-
24
- - Fail if using a bad version of scout_apm [#5616]
25
- - Add pagination to Busy page [#5556]
26
- - Speed up WorkSet#each [#5559]
27
8
  - Adjust CurrentAttributes to work with the String class name so we aren't referencing
28
9
  the Class within a Rails initializer [#5536]
29
10
 
@@ -595,7 +576,7 @@ Sidekiq::Middleware::Server::Logging -> Sidekiq::JobLogger
595
576
  - The `SomeWorker.set(options)` API was re-written to avoid thread-local state. [#2152]
596
577
  - Sidekiq Enterprise's encrypted jobs now display "[encrypted data]" in the Web UI instead
597
578
  of random hex bytes.
598
- - Please see the [5.0 Upgrade notes](5.0-Upgrade.md) for more detail.
579
+ - Please see the [5.0 Upgrade notes](docs/5.0-Upgrade.md) for more detail.
599
580
 
600
581
  4.2.10
601
582
  -----------
@@ -813,7 +794,7 @@ Sidekiq::Queues.clear_all
813
794
  - Sidekiq's internals have been completely overhauled for performance
814
795
  and to remove dependencies. This has resulted in major speedups, as
815
796
  [detailed on my blog](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/).
816
- - See the [4.0 upgrade notes](4.0-Upgrade.md) for more detail.
797
+ - See the [4.0 upgrade notes](docs/4.0-Upgrade.md) for more detail.
817
798
 
818
799
  3.5.4
819
800
  -----------
@@ -1080,7 +1061,7 @@ sidekiq_options :dead => false, :retry => 5
1080
1061
  3.0.0
1081
1062
  -----------
1082
1063
 
1083
- Please see [3.0-Upgrade.md](3.0-Upgrade.md) for more comprehensive upgrade notes.
1064
+ Please see [3.0-Upgrade.md](docs/3.0-Upgrade.md) for more comprehensive upgrade notes.
1084
1065
 
1085
1066
  - **Dead Job Queue** - jobs which run out of retries are now moved to a dead
1086
1067
  job queue. These jobs must be retried manually or they will expire
@@ -1124,7 +1105,7 @@ Sidekiq::Client.via(ConnectionPool.new { Redis.new }) do
1124
1105
  end
1125
1106
  ```
1126
1107
  **Sharding support does require a breaking change to client-side
1127
- middleware, see 3.0-Upgrade.md.**
1108
+ middleware, see docs/3.0-Upgrade.md.**
1128
1109
  - New Chinese, Greek, Swedish and Czech translations for the Web UI.
1129
1110
  - Updated most languages translations for the new UI features.
1130
1111
  - **Remove official Capistrano integration** - this integration has been
data/README.md CHANGED
@@ -27,10 +27,10 @@ This benchmark can be found in `bin/sidekiqload` and assumes a Redis network lat
27
27
  Requirements
28
28
  -----------------
29
29
 
30
- - Redis: 4.0+
31
- - Ruby: MRI 2.5+ or JRuby 9.2+.
30
+ - Redis: 6.2+
31
+ - Ruby: MRI 2.7+ or JRuby 9.3+.
32
32
 
33
- Sidekiq 6.0 supports Rails 5.0+ but does not require it.
33
+ Sidekiq 7.0 supports Rails 6.0+ but does not require it.
34
34
 
35
35
 
36
36
  Installation
@@ -52,24 +52,26 @@ Sidekiq and see its features in action. Here's the Web UI:
52
52
  Want to Upgrade?
53
53
  -------------------
54
54
 
55
+ Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
56
+ Upgrade notes between each major version can be found in the `docs/` directory.
57
+
55
58
  I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
56
59
  features, a commercial-friendly license and allow you to support high
57
60
  quality open source development all at the same time. Please see the
58
61
  [Sidekiq](https://sidekiq.org/) homepage for more detail.
59
62
 
60
- Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
61
- features and changes to Sidekiq and its bigger siblings.
62
-
63
63
 
64
64
  Problems?
65
65
  -----------------
66
66
 
67
- **Please do not directly email any Sidekiq committers with questions or problems.** A community is best served when discussions are held in public.
67
+ **Please do not directly email any Sidekiq committers with questions or problems.**
68
+ A community is best served when discussions are held in public.
68
69
 
69
70
  If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
70
71
  Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
71
72
 
72
- Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at https://sidekiq.org; email support@contribsys.com for help.
73
+ Sidekiq Pro and Sidekiq Enterprise customers get private email support.
74
+ You can purchase at https://sidekiq.org; email support@contribsys.com for help.
73
75
 
74
76
  Useful resources:
75
77
 
@@ -77,7 +79,7 @@ Useful resources:
77
79
  * Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
78
80
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
79
81
 
80
- Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
82
+ Every Friday morning is Sidekiq office hour: I video chat and answer questions.
81
83
  See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
82
84
 
83
85
  Contributing
@@ -85,12 +87,11 @@ Contributing
85
87
 
86
88
  Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
87
89
 
88
-
89
90
  License
90
91
  -----------------
91
92
 
92
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
93
-
93
+ Please see [LICENSE.txt](https://github.com/mperham/sidekiq/blob/main/LICENSE.txt) for licensing details.
94
+ The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/mperham/sidekiq/blob/main/COMM-LICENSE.txt).
94
95
 
95
96
  Author
96
97
  -----------------
data/bin/sidekiq CHANGED
@@ -10,7 +10,7 @@ def integrate_with_systemd
10
10
  return unless ENV["NOTIFY_SOCKET"]
11
11
 
12
12
  Sidekiq.configure_server do |config|
13
- Sidekiq.logger.info "Enabling systemd notification integration"
13
+ config.logger.info "Enabling systemd notification integration"
14
14
  require "sidekiq/sd_notify"
15
15
  config.on(:startup) do
16
16
  Sidekiq::SdNotify.ready
@@ -31,12 +31,7 @@ begin
31
31
  cli.run
32
32
  rescue => e
33
33
  raise e if $DEBUG
34
- if Sidekiq.error_handlers.length == 0
35
- warn e.message
36
- warn e.backtrace.join("\n")
37
- else
38
- cli.handle_exception e
39
- end
40
-
34
+ warn e.message
35
+ warn e.backtrace.join("\n")
41
36
  exit 1
42
37
  end
data/bin/sidekiqload CHANGED
@@ -8,18 +8,11 @@ $TESTING = false
8
8
  require "bundler/setup"
9
9
  Bundler.require(:default, :load_test)
10
10
 
11
- require_relative "../lib/sidekiq/cli"
12
- require_relative "../lib/sidekiq/launcher"
13
-
14
- if ENV["SIDEKIQ_REDIS_CLIENT"]
15
- Sidekiq::RedisConnection.adapter = :redis_client
16
- end
17
-
18
- Sidekiq.configure_server do |config|
19
- config.options[:concurrency] = 10
11
+ x = Sidekiq.configure_embed do |config|
20
12
  config.redis = {db: 13, port: 6380}
13
+ config.concurrency = 10
21
14
  # config.redis = { db: 13, port: 6380, driver: :hiredis}
22
- config.options[:queues] << "default"
15
+ config.queues = %w[default]
23
16
  config.logger.level = Logger::ERROR
24
17
  config.average_scheduled_poll_interval = 2
25
18
  config.reliable! if defined?(Sidekiq::Pro)
@@ -101,8 +94,8 @@ Sidekiq.logger.error "Created #{count * iter} jobs"
101
94
  start = Time.now
102
95
 
103
96
  Monitoring = Thread.new do
104
- while true
105
- sleep 0.2
97
+ loop do
98
+ sleep 1.0
106
99
  qsize = Sidekiq.redis do |conn|
107
100
  conn.llen "queue:default"
108
101
  end
@@ -132,30 +125,28 @@ def with_latency(latency, &block)
132
125
  end
133
126
 
134
127
  begin
135
- # RubyProf::exclude_threads = [ Monitoring ]
128
+ # RubyProf.exclude_threads = [Monitoring]
136
129
  # RubyProf.start
137
- events = Sidekiq.options[:lifecycle_events][:startup]
138
- events.each(&:call)
139
- events.clear
140
-
130
+
141
131
  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
142
- launcher = Sidekiq::Launcher.new(Sidekiq)
143
- launcher.run
132
+ x.run
144
133
 
145
- while readable_io = IO.select([self_read])
134
+ while (readable_io = IO.select([self_read]))
146
135
  signal = readable_io.first[0].gets.strip
147
- handle_signal(launcher, signal)
136
+ handle_signal(x, signal)
148
137
  end
149
138
  end
150
- rescue SystemExit => e
139
+ rescue SystemExit
151
140
  # Sidekiq.logger.error("Profiling...")
152
141
  # result = RubyProf.stop
153
142
  # printer = RubyProf::GraphHtmlPrinter.new(result)
154
- # printer.print(File.new("output.html", "w"), :min_percent => 1)
143
+ # printer.print(File.new("output.html", "w"), min_percent: 1)
155
144
  # normal
156
145
  rescue => e
157
146
  raise e if $DEBUG
158
147
  warn e.message
159
148
  warn e.backtrace.join("\n")
160
149
  exit 1
150
+ ensure
151
+ x.stop
161
152
  end
data/lib/sidekiq/api.rb CHANGED
@@ -6,10 +6,7 @@ require "zlib"
6
6
  require "set"
7
7
  require "base64"
8
8
 
9
- if ENV["SIDEKIQ_METRICS_BETA"]
10
- require "sidekiq/metrics/deploy"
11
- require "sidekiq/metrics/query"
12
- end
9
+ require "sidekiq/metrics/query"
13
10
 
14
11
  #
15
12
  # Sidekiq's Data API provides a Ruby object model on top
@@ -70,7 +67,18 @@ module Sidekiq
70
67
  end
71
68
 
72
69
  def queues
73
- Sidekiq::Stats::Queues.new.lengths
70
+ Sidekiq.redis do |conn|
71
+ queues = conn.sscan("queues").to_a
72
+
73
+ lengths = conn.pipelined { |pipeline|
74
+ queues.each do |queue|
75
+ pipeline.llen("queue:#{queue}")
76
+ end
77
+ }
78
+
79
+ array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
80
+ array_of_arrays.to_h
81
+ end
74
82
  end
75
83
 
76
84
  # O(1) redis calls
@@ -117,11 +125,11 @@ module Sidekiq
117
125
  # @api private
118
126
  def fetch_stats_slow!
119
127
  processes = Sidekiq.redis { |conn|
120
- conn.sscan_each("processes").to_a
128
+ conn.sscan("processes").to_a
121
129
  }
122
130
 
123
131
  queues = Sidekiq.redis { |conn|
124
- conn.sscan_each("queues").to_a
132
+ conn.sscan("queues").to_a
125
133
  }
126
134
 
127
135
  pipe2_res = Sidekiq.redis { |conn|
@@ -133,7 +141,7 @@ module Sidekiq
133
141
 
134
142
  s = processes.size
135
143
  workers_size = pipe2_res[0...s].sum(&:to_i)
136
- enqueued = pipe2_res[s..-1].sum(&:to_i)
144
+ enqueued = pipe2_res[s..].sum(&:to_i)
137
145
 
138
146
  @stats[:workers_size] = workers_size
139
147
  @stats[:enqueued] = enqueued
@@ -168,25 +176,8 @@ module Sidekiq
168
176
  @stats[s] || raise(ArgumentError, "Unknown stat #{s}")
169
177
  end
170
178
 
171
- class Queues
172
- def lengths
173
- Sidekiq.redis do |conn|
174
- queues = conn.sscan_each("queues").to_a
175
-
176
- lengths = conn.pipelined { |pipeline|
177
- queues.each do |queue|
178
- pipeline.llen("queue:#{queue}")
179
- end
180
- }
181
-
182
- array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
183
- array_of_arrays.to_h
184
- end
185
- end
186
- end
187
-
188
179
  class History
189
- def initialize(days_previous, start_date = nil)
180
+ def initialize(days_previous, start_date = nil, pool: nil)
190
181
  # we only store five years of data in Redis
191
182
  raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
192
183
  @days_previous = days_previous
@@ -211,15 +202,10 @@ module Sidekiq
211
202
 
212
203
  keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
213
204
 
214
- begin
215
- Sidekiq.redis do |conn|
216
- conn.mget(keys).each_with_index do |value, idx|
217
- stat_hash[dates[idx]] = value ? value.to_i : 0
218
- end
205
+ Sidekiq.redis do |conn|
206
+ conn.mget(keys).each_with_index do |value, idx|
207
+ stat_hash[dates[idx]] = value ? value.to_i : 0
219
208
  end
220
- rescue RedisConnection.adapter::CommandError
221
- # mget will trigger a CROSSSLOT error when run against a Cluster
222
- # TODO Someone want to add Cluster support?
223
209
  end
224
210
 
225
211
  stat_hash
@@ -247,7 +233,7 @@ module Sidekiq
247
233
  #
248
234
  # @return [Array<Sidekiq::Queue>]
249
235
  def self.all
250
- Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
236
+ Sidekiq.redis { |c| c.sscan("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
251
237
  end
252
238
 
253
239
  attr_reader :name
@@ -388,12 +374,7 @@ module Sidekiq
388
374
  def display_class
389
375
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
390
376
  @klass ||= self["display_class"] || begin
391
- case klass
392
- when /\ASidekiq::Extensions::Delayed/
393
- safe_load(args[0], klass) do |target, method, _|
394
- "#{target}.#{method}"
395
- end
396
- when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
377
+ if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
397
378
  job_class = @item["wrapped"] || args[0]
398
379
  if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
399
380
  # MailerClass#mailer_method
@@ -409,16 +390,7 @@ module Sidekiq
409
390
 
410
391
  def display_args
411
392
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
412
- @display_args ||= case klass
413
- when /\ASidekiq::Extensions::Delayed/
414
- safe_load(args[0], args) do |_, _, arg, kwarg|
415
- if !kwarg || kwarg.empty?
416
- arg
417
- else
418
- [arg, kwarg]
419
- end
420
- end
421
- when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
393
+ @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
422
394
  job_args = self["wrapped"] ? args[0]["arguments"] : []
423
395
  if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
424
396
  # remove MailerClass, mailer_method and 'deliver_now'
@@ -491,31 +463,10 @@ module Sidekiq
491
463
 
492
464
  private
493
465
 
494
- def safe_load(content, default)
495
- yield(*YAML.load(content))
496
- rescue => ex
497
- # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
498
- # memory yet so the YAML can't be loaded.
499
- # TODO is this still necessary? Zeitwerk reloader should handle?
500
- Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
501
- default
502
- end
503
-
504
466
  def uncompress_backtrace(backtrace)
505
- if backtrace.is_a?(Array)
506
- # Handle old jobs with raw Array backtrace format
507
- backtrace
508
- else
509
- decoded = Base64.decode64(backtrace)
510
- uncompressed = Zlib::Inflate.inflate(decoded)
511
- begin
512
- Sidekiq.load_json(uncompressed)
513
- rescue
514
- # Handle old jobs with marshalled backtrace format
515
- # TODO Remove in 7.x
516
- Marshal.load(uncompressed)
517
- end
518
- end
467
+ decoded = Base64.decode64(backtrace)
468
+ uncompressed = Zlib::Inflate.inflate(decoded)
469
+ Sidekiq.load_json(uncompressed)
519
470
  end
520
471
  end
521
472
 
@@ -656,7 +607,7 @@ module Sidekiq
656
607
 
657
608
  match = "*#{match}*" unless match.include?("*")
658
609
  Sidekiq.redis do |conn|
659
- conn.zscan_each(name, match: match, count: count) do |entry, score|
610
+ conn.zscan(name, match: match, count: count) do |entry, score|
660
611
  yield SortedEntry.new(self, score, entry)
661
612
  end
662
613
  end
@@ -746,7 +697,7 @@ module Sidekiq
746
697
  # @return [SortedEntry] the record or nil
747
698
  def find_job(jid)
748
699
  Sidekiq.redis do |conn|
749
- conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
700
+ conn.zscan(name, match: "*#{jid}*", count: 100) do |entry, score|
750
701
  job = JSON.parse(entry)
751
702
  matched = job["jid"] == jid
752
703
  return SortedEntry.new(self, score, entry) if matched
@@ -792,12 +743,8 @@ module Sidekiq
792
743
  # example where I'm selecting jobs based on some complex logic
793
744
  # and deleting them from the scheduled set.
794
745
  #
795
- # r = Sidekiq::ScheduledSet.new
796
- # r.select do |scheduled|
797
- # scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
798
- # scheduled.args[0] == 'User' &&
799
- # scheduled.args[1] == 'setup_new_subscriber'
800
- # end.map(&:delete)
746
+ # See the API wiki page for usage notes and examples.
747
+ #
801
748
  class ScheduledSet < JobSet
802
749
  def initialize
803
750
  super "schedule"
@@ -810,12 +757,8 @@ module Sidekiq
810
757
  # example where I'm selecting all jobs of a certain type
811
758
  # and deleting them from the retry queue.
812
759
  #
813
- # r = Sidekiq::RetrySet.new
814
- # r.select do |retri|
815
- # retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
816
- # retri.args[0] == 'User' &&
817
- # retri.args[1] == 'setup_new_subscriber'
818
- # end.map(&:delete)
760
+ # See the API wiki page for usage notes and examples.
761
+ #
819
762
  class RetrySet < JobSet
820
763
  def initialize
821
764
  super "retry"
@@ -849,8 +792,8 @@ module Sidekiq
849
792
  Sidekiq.redis do |conn|
850
793
  conn.multi do |transaction|
851
794
  transaction.zadd(name, now.to_s, message)
852
- transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
853
- transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
795
+ transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
796
+ transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
854
797
  end
855
798
  end
856
799
 
@@ -858,7 +801,7 @@ module Sidekiq
858
801
  job = Sidekiq.load_json(message)
859
802
  r = RuntimeError.new("Job killed by API")
860
803
  r.set_backtrace(caller)
861
- Sidekiq.death_handlers.each do |handle|
804
+ Sidekiq.default_configuration.death_handlers.each do |handle|
862
805
  handle.call(job, r)
863
806
  end
864
807
  end
@@ -869,18 +812,6 @@ module Sidekiq
869
812
  def retry_all
870
813
  each(&:retry) while size > 0
871
814
  end
872
-
873
- # The maximum size of the Dead set. Older entries will be trimmed
874
- # to stay within this limit. Default value is 10,000.
875
- def self.max_jobs
876
- Sidekiq[:dead_max_jobs]
877
- end
878
-
879
- # The time limit for entries within the Dead set. Older entries will be thrown away.
880
- # Default value is six months.
881
- def self.timeout
882
- Sidekiq[:dead_timeout_in_seconds]
883
- end
884
815
  end
885
816
 
886
817
  ##
@@ -909,7 +840,7 @@ module Sidekiq
909
840
 
910
841
  count = 0
911
842
  Sidekiq.redis do |conn|
912
- procs = conn.sscan_each("processes").to_a
843
+ procs = conn.sscan("processes").to_a
913
844
  heartbeats = conn.pipelined { |pipeline|
914
845
  procs.each do |key|
915
846
  pipeline.hget(key, "info")
@@ -929,7 +860,7 @@ module Sidekiq
929
860
 
930
861
  def each
931
862
  result = Sidekiq.redis { |conn|
932
- procs = conn.sscan_each("processes").to_a.sort
863
+ procs = conn.sscan("processes").to_a.sort
933
864
 
934
865
  # We're making a tradeoff here between consuming more memory instead of
935
866
  # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
@@ -1021,7 +952,7 @@ module Sidekiq
1021
952
  end
1022
953
 
1023
954
  def labels
1024
- Array(self["labels"])
955
+ self["labels"].to_a
1025
956
  end
1026
957
 
1027
958
  def [](key)
@@ -1103,31 +1034,24 @@ module Sidekiq
1103
1034
 
1104
1035
  def each(&block)
1105
1036
  results = []
1106
- procs = nil
1107
- all_works = nil
1108
-
1109
1037
  Sidekiq.redis do |conn|
1110
- procs = conn.sscan_each("processes").to_a.sort
1111
-
1112
- all_works = conn.pipelined do |pipeline|
1113
- procs.each do |key|
1038
+ procs = conn.sscan("processes").to_a
1039
+ procs.sort.each do |key|
1040
+ valid, workers = conn.pipelined { |pipeline|
1041
+ pipeline.exists(key)
1114
1042
  pipeline.hgetall("#{key}:work")
1043
+ }
1044
+ next unless valid > 0
1045
+ workers.each_pair do |tid, json|
1046
+ hsh = Sidekiq.load_json(json)
1047
+ p = hsh["payload"]
1048
+ # avoid breaking API, this is a side effect of the JSON optimization in #4316
1049
+ hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
1050
+ results << [key, tid, hsh]
1115
1051
  end
1116
1052
  end
1117
1053
  end
1118
1054
 
1119
- procs.zip(all_works).each do |key, workers|
1120
- workers.each_pair do |tid, json|
1121
- next if json.empty?
1122
-
1123
- hsh = Sidekiq.load_json(json)
1124
- p = hsh["payload"]
1125
- # avoid breaking API, this is a side effect of the JSON optimization in #4316
1126
- hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
1127
- results << [key, tid, hsh]
1128
- end
1129
- end
1130
-
1131
1055
  results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
1132
1056
  end
1133
1057
 
@@ -1139,7 +1063,7 @@ module Sidekiq
1139
1063
  # which can easily get out of sync with crashy processes.
1140
1064
  def size
1141
1065
  Sidekiq.redis do |conn|
1142
- procs = conn.sscan_each("processes").to_a
1066
+ procs = conn.sscan("processes").to_a
1143
1067
  if procs.empty?
1144
1068
  0
1145
1069
  else
@@ -0,0 +1,110 @@
1
+ require "sidekiq/component"
2
+
3
+ module Sidekiq
4
+ # A Sidekiq::Capsule is the set of resources necessary to
5
+ # process one or more queues with a given concurrency.
6
+ # One "default" Capsule is started but the user may declare additional
7
+ # Capsules in the initializer.
8
+ #
9
+ # To process a "single" queue with one thread so jobs are processed
10
+ # serially, you can do this:
11
+ #
12
+ # Sidekiq.configure_server do |config|
13
+ # config.capsule("single-threaded") do |cap|
14
+ # cap.concurrency = 1
15
+ # cap.queues = %w(single)
16
+ # end
17
+ # end
18
+ class Capsule
19
+ include Sidekiq::Component
20
+
21
+ attr_reader :name
22
+ attr_reader :queues
23
+ attr_accessor :concurrency
24
+
25
+ def initialize(name, config)
26
+ @name = name
27
+ @config = config
28
+ @queues = ["default"]
29
+ @concurrency = config[:concurrency]
30
+ end
31
+
32
+ def fetcher
33
+ @fetcher ||= begin
34
+ inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
35
+ inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
36
+ inst
37
+ end
38
+ end
39
+
40
+ def stop
41
+ fetcher&.bulk_requeue([], nil)
42
+ end
43
+
44
+ def queues=(val)
45
+ @queues = Array(val).each_with_object([]) do |qstr, memo|
46
+ arr = qstr
47
+ arr = qstr.split(",") if qstr.is_a?(String)
48
+ name, weight = arr
49
+ [weight.to_i, 1].max.times do
50
+ memo << name
51
+ end
52
+ end
53
+ end
54
+
55
+ # Allow the middleware to be different per-capsule.
56
+ # Avoid if possible and add middleware globally so all
57
+ # capsules share the same chains. Easier to debug that way.
58
+ def client_middleware
59
+ @client_chain ||= config.client_middleware.copy_for(self)
60
+ yield @client_chain if block_given?
61
+ @client_chain
62
+ end
63
+
64
+ def server_middleware
65
+ @server_chain ||= config.server_middleware.copy_for(self)
66
+ yield @server_chain if block_given?
67
+ @server_chain
68
+ end
69
+
70
+ def redis_pool
71
+ Thread.current[:sidekiq_redis_pool] || local_redis_pool
72
+ end
73
+
74
+ def local_redis_pool
75
+ # connection pool is lazy, it will not create connections unless you actually need them
76
+ # so don't be skimpy!
77
+ @redis ||= config.new_redis_pool(@concurrency, name)
78
+ end
79
+
80
+ def redis
81
+ raise ArgumentError, "requires a block" unless block_given?
82
+ redis_pool.with do |conn|
83
+ retryable = true
84
+ begin
85
+ yield conn
86
+ rescue RedisClientAdapter::BaseError => ex
87
+ # 2550 Failover can cause the server to become a replica, need
88
+ # to disconnect and reopen the socket to get back to the primary.
89
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
90
+ # 4985 Use the same logic when a blocking command is force-unblocked
91
+ # The same retry logic is also used in client.rb
92
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
93
+ conn.close
94
+ retryable = false
95
+ retry
96
+ end
97
+ raise
98
+ end
99
+ end
100
+ end
101
+
102
+ def lookup(name)
103
+ config.lookup(name)
104
+ end
105
+
106
+ def logger
107
+ config.logger
108
+ end
109
+ end
110
+ end