sidekiq 6.5.1 → 7.3.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +376 -12
  3. data/README.md +43 -35
  4. data/bin/multi_queue_bench +271 -0
  5. data/bin/sidekiq +3 -8
  6. data/bin/sidekiqload +213 -118
  7. data/bin/sidekiqmon +3 -0
  8. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +88 -0
  9. data/lib/generators/sidekiq/job_generator.rb +2 -0
  10. data/lib/sidekiq/api.rb +378 -173
  11. data/lib/sidekiq/capsule.rb +132 -0
  12. data/lib/sidekiq/cli.rb +61 -63
  13. data/lib/sidekiq/client.rb +89 -40
  14. data/lib/sidekiq/component.rb +6 -2
  15. data/lib/sidekiq/config.rb +305 -0
  16. data/lib/sidekiq/deploy.rb +64 -0
  17. data/lib/sidekiq/embedded.rb +63 -0
  18. data/lib/sidekiq/fetch.rb +11 -14
  19. data/lib/sidekiq/iterable_job.rb +55 -0
  20. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  21. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  22. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  23. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  24. data/lib/sidekiq/job/iterable.rb +294 -0
  25. data/lib/sidekiq/job.rb +382 -10
  26. data/lib/sidekiq/job_logger.rb +8 -7
  27. data/lib/sidekiq/job_retry.rb +89 -46
  28. data/lib/sidekiq/job_util.rb +53 -15
  29. data/lib/sidekiq/launcher.rb +77 -69
  30. data/lib/sidekiq/logger.rb +2 -27
  31. data/lib/sidekiq/manager.rb +9 -11
  32. data/lib/sidekiq/metrics/query.rb +158 -0
  33. data/lib/sidekiq/metrics/shared.rb +106 -0
  34. data/lib/sidekiq/metrics/tracking.rb +148 -0
  35. data/lib/sidekiq/middleware/chain.rb +84 -48
  36. data/lib/sidekiq/middleware/current_attributes.rb +87 -20
  37. data/lib/sidekiq/middleware/modules.rb +2 -0
  38. data/lib/sidekiq/monitor.rb +19 -5
  39. data/lib/sidekiq/paginator.rb +11 -3
  40. data/lib/sidekiq/processor.rb +67 -56
  41. data/lib/sidekiq/rails.rb +22 -16
  42. data/lib/sidekiq/redis_client_adapter.rb +31 -71
  43. data/lib/sidekiq/redis_connection.rb +44 -117
  44. data/lib/sidekiq/ring_buffer.rb +2 -0
  45. data/lib/sidekiq/scheduled.rb +62 -35
  46. data/lib/sidekiq/systemd.rb +2 -0
  47. data/lib/sidekiq/testing.rb +37 -46
  48. data/lib/sidekiq/transaction_aware_client.rb +11 -5
  49. data/lib/sidekiq/version.rb +6 -1
  50. data/lib/sidekiq/web/action.rb +15 -5
  51. data/lib/sidekiq/web/application.rb +94 -24
  52. data/lib/sidekiq/web/csrf_protection.rb +10 -7
  53. data/lib/sidekiq/web/helpers.rb +118 -45
  54. data/lib/sidekiq/web/router.rb +5 -2
  55. data/lib/sidekiq/web.rb +67 -15
  56. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  57. data/lib/sidekiq.rb +78 -266
  58. data/sidekiq.gemspec +12 -10
  59. data/web/assets/javascripts/application.js +46 -1
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +192 -0
  64. data/web/assets/javascripts/dashboard.js +11 -250
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +4 -0
  67. data/web/assets/stylesheets/application-rtl.css +10 -89
  68. data/web/assets/stylesheets/application.css +98 -295
  69. data/web/locales/ar.yml +70 -70
  70. data/web/locales/cs.yml +62 -62
  71. data/web/locales/da.yml +60 -53
  72. data/web/locales/de.yml +65 -65
  73. data/web/locales/el.yml +43 -24
  74. data/web/locales/en.yml +83 -69
  75. data/web/locales/es.yml +68 -68
  76. data/web/locales/fa.yml +65 -65
  77. data/web/locales/fr.yml +80 -67
  78. data/web/locales/gd.yml +98 -0
  79. data/web/locales/he.yml +65 -64
  80. data/web/locales/hi.yml +59 -59
  81. data/web/locales/it.yml +85 -54
  82. data/web/locales/ja.yml +72 -68
  83. data/web/locales/ko.yml +52 -52
  84. data/web/locales/lt.yml +66 -66
  85. data/web/locales/nb.yml +61 -61
  86. data/web/locales/nl.yml +52 -52
  87. data/web/locales/pl.yml +45 -45
  88. data/web/locales/pt-br.yml +78 -69
  89. data/web/locales/pt.yml +51 -51
  90. data/web/locales/ru.yml +67 -66
  91. data/web/locales/sv.yml +53 -53
  92. data/web/locales/ta.yml +60 -60
  93. data/web/locales/tr.yml +100 -0
  94. data/web/locales/uk.yml +85 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +67 -67
  97. data/web/locales/zh-cn.yml +42 -16
  98. data/web/locales/zh-tw.yml +41 -8
  99. data/web/views/_footer.erb +17 -2
  100. data/web/views/_job_info.erb +18 -2
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +1 -1
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +1 -1
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +49 -33
  107. data/web/views/dashboard.erb +28 -6
  108. data/web/views/filtering.erb +6 -0
  109. data/web/views/layout.erb +6 -6
  110. data/web/views/metrics.erb +90 -0
  111. data/web/views/metrics_for_job.erb +59 -0
  112. data/web/views/morgue.erb +5 -9
  113. data/web/views/queue.erb +15 -15
  114. data/web/views/queues.erb +9 -3
  115. data/web/views/retries.erb +5 -9
  116. data/web/views/scheduled.erb +12 -13
  117. metadata +61 -26
  118. data/lib/sidekiq/.DS_Store +0 -0
  119. data/lib/sidekiq/delay.rb +0 -43
  120. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  121. data/lib/sidekiq/extensions/active_record.rb +0 -43
  122. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  123. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  124. data/lib/sidekiq/worker.rb +0 -367
  125. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
- require "sidekiq/api"
5
4
  require "sidekiq/component"
6
5
 
7
6
  module Sidekiq
@@ -9,16 +8,20 @@ module Sidekiq
9
8
  SETS = %w[retry schedule]
10
9
 
11
10
  class Enq
11
+ include Sidekiq::Component
12
+
12
13
  LUA_ZPOPBYSCORE = <<~LUA
13
14
  local key, now = KEYS[1], ARGV[1]
14
- local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
15
+ local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
15
16
  if jobs[1] then
16
17
  redis.call("zrem", key, jobs[1])
17
18
  return jobs[1]
18
19
  end
19
20
  LUA
20
21
 
21
- def initialize
22
+ def initialize(container)
23
+ @config = container
24
+ @client = Sidekiq::Client.new(config: container)
22
25
  @done = false
23
26
  @lua_zpopbyscore_sha = nil
24
27
  end
@@ -26,15 +29,15 @@ module Sidekiq
26
29
  def enqueue_jobs(sorted_sets = SETS)
27
30
  # A job's "score" in Redis is the time at which it should be processed.
28
31
  # Just check Redis for the set of jobs with a timestamp before now.
29
- Sidekiq.redis do |conn|
32
+ redis do |conn|
30
33
  sorted_sets.each do |sorted_set|
31
34
  # Get next item in the queue with score (time to execute) <= now.
32
35
  # We need to go through the list one at a time to reduce the risk of something
33
36
  # going wrong between the time jobs are popped from the scheduled queue and when
34
37
  # they are pushed onto a work queue and losing the jobs.
35
38
  while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
36
- Sidekiq::Client.push(Sidekiq.load_json(job))
37
- Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
39
+ @client.push(Sidekiq.load_json(job))
40
+ logger.debug { "enqueued #{sorted_set}: #{job}" }
38
41
  end
39
42
  end
40
43
  end
@@ -48,12 +51,11 @@ module Sidekiq
48
51
 
49
52
  def zpopbyscore(conn, keys: nil, argv: nil)
50
53
  if @lua_zpopbyscore_sha.nil?
51
- raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
52
- @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
54
+ @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
53
55
  end
54
56
 
55
- conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
56
- rescue RedisConnection.adapter::CommandError => e
57
+ conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
58
+ rescue RedisClient::CommandError => e
57
59
  raise unless e.message.start_with?("NOSCRIPT")
58
60
 
59
61
  @lua_zpopbyscore_sha = nil
@@ -71,9 +73,9 @@ module Sidekiq
71
73
 
72
74
  INITIAL_WAIT = 10
73
75
 
74
- def initialize(options)
75
- @config = options
76
- @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
76
+ def initialize(config)
77
+ @config = config
78
+ @enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
77
79
  @sleeper = ConnectionPool::TimedStack.new
78
80
  @done = false
79
81
  @thread = nil
@@ -83,14 +85,10 @@ module Sidekiq
83
85
  # Shut down this instance, will pause until the thread is dead.
84
86
  def terminate
85
87
  @done = true
86
- @enq.terminate if @enq.respond_to?(:terminate)
88
+ @enq.terminate
87
89
 
88
- if @thread
89
- t = @thread
90
- @thread = nil
91
- @sleeper << 0
92
- t.value
93
- end
90
+ @sleeper << 0
91
+ @thread&.value
94
92
  end
95
93
 
96
94
  def start
@@ -146,15 +144,18 @@ module Sidekiq
146
144
  # In the example above, each process should schedule every 10 seconds on average. We special
147
145
  # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
148
146
  # As we run more processes, the scheduling interval average will approach an even spread
149
- # between 0 and poll interval so we don't need this artifical boost.
147
+ # between 0 and poll interval so we don't need this artificial boost.
150
148
  #
151
- if process_count < 10
149
+ count = process_count
150
+ interval = poll_interval_average(count)
151
+
152
+ if count < 10
152
153
  # For small clusters, calculate a random interval that is ±50% the desired average.
153
- poll_interval_average * rand + poll_interval_average.to_f / 2
154
+ interval * rand + interval.to_f / 2
154
155
  else
155
156
  # With 10+ processes, we should have enough randomness to get decent polling
156
157
  # across the entire timespan
157
- poll_interval_average * rand
158
+ interval * rand
158
159
  end
159
160
  end
160
161
 
@@ -171,31 +172,52 @@ module Sidekiq
171
172
  # the same time: the thundering herd problem.
172
173
  #
173
174
  # We only do this if poll_interval_average is unset (the default).
174
- def poll_interval_average
175
- @config[:poll_interval_average] ||= scaled_poll_interval
175
+ def poll_interval_average(count)
176
+ @config[:poll_interval_average] || scaled_poll_interval(count)
176
177
  end
177
178
 
178
179
  # Calculates an average poll interval based on the number of known Sidekiq processes.
179
180
  # This minimizes a single point of failure by dispersing check-ins but without taxing
180
181
  # Redis if you run many Sidekiq processes.
181
- def scaled_poll_interval
182
+ def scaled_poll_interval(process_count)
182
183
  process_count * @config[:average_scheduled_poll_interval]
183
184
  end
184
185
 
185
186
  def process_count
186
- # The work buried within Sidekiq::ProcessSet#cleanup can be
187
- # expensive at scale. Cut it down by 90% with this counter.
188
- # NB: This method is only called by the scheduler thread so we
189
- # don't need to worry about the thread safety of +=.
190
- pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
187
+ pcount = Sidekiq.redis { |conn| conn.scard("processes") }
191
188
  pcount = 1 if pcount == 0
192
- @count_calls += 1
193
189
  pcount
194
190
  end
195
191
 
192
+ # A copy of Sidekiq::ProcessSet#cleanup because server
193
+ # should never depend on sidekiq/api.
194
+ def cleanup
195
+ # dont run cleanup more than once per minute
196
+ return 0 unless redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
197
+
198
+ count = 0
199
+ redis do |conn|
200
+ procs = conn.sscan("processes").to_a
201
+ heartbeats = conn.pipelined { |pipeline|
202
+ procs.each do |key|
203
+ pipeline.hget(key, "info")
204
+ end
205
+ }
206
+
207
+ # the hash named key has an expiry of 60 seconds.
208
+ # if it's not found, that means the process has not reported
209
+ # in to Redis and probably died.
210
+ to_prune = procs.select.with_index { |proc, i|
211
+ heartbeats[i].nil?
212
+ }
213
+ count = conn.srem("processes", to_prune) unless to_prune.empty?
214
+ end
215
+ count
216
+ end
217
+
196
218
  def initial_wait
197
- # Have all processes sleep between 5-15 seconds. 10 seconds
198
- # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
219
+ # Have all processes sleep between 5-15 seconds. 10 seconds to give time for
220
+ # the heartbeat to register (if the poll interval is going to be calculated by the number
199
221
  # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
200
222
  total = 0
201
223
  total += INITIAL_WAIT unless @config[:poll_interval_average]
@@ -203,6 +225,11 @@ module Sidekiq
203
225
 
204
226
  @sleeper.pop(total)
205
227
  rescue Timeout::Error
228
+ ensure
229
+ # periodically clean out the `processes` set in Redis which can collect
230
+ # references to dead processes over time. The process count affects how
231
+ # often we scan for scheduled jobs.
232
+ cleanup
206
233
  end
207
234
  end
208
235
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Sidekiq's systemd integration allows Sidekiq to inform systemd:
3
5
  # 1. when it has successfully started
@@ -5,23 +5,42 @@ require "sidekiq"
5
5
 
6
6
  module Sidekiq
7
7
  class Testing
8
+ class TestModeAlreadySetError < RuntimeError; end
8
9
  class << self
9
- attr_accessor :__test_mode
10
+ attr_accessor :__global_test_mode
10
11
 
12
+ # Calling without a block sets the global test mode, affecting
13
+ # all threads. Calling with a block only affects the current Thread.
11
14
  def __set_test_mode(mode)
12
15
  if block_given?
13
- current_mode = __test_mode
16
+ # Reentrant testing modes will lead to a rat's nest of code which is
17
+ # hard to reason about. You can set the testing mode once globally and
18
+ # you can override that global setting once per-thread.
19
+ raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
20
+
21
+ self.__local_test_mode = mode
14
22
  begin
15
- self.__test_mode = mode
16
23
  yield
17
24
  ensure
18
- self.__test_mode = current_mode
25
+ self.__local_test_mode = nil
19
26
  end
20
27
  else
21
- self.__test_mode = mode
28
+ self.__global_test_mode = mode
22
29
  end
23
30
  end
24
31
 
32
+ def __test_mode
33
+ __local_test_mode || __global_test_mode
34
+ end
35
+
36
+ def __local_test_mode
37
+ Thread.current[:__sidekiq_test_mode]
38
+ end
39
+
40
+ def __local_test_mode=(value)
41
+ Thread.current[:__sidekiq_test_mode] = value
42
+ end
43
+
25
44
  def disable!(&block)
26
45
  __set_test_mode(:disable, &block)
27
46
  end
@@ -51,19 +70,10 @@ module Sidekiq
51
70
  end
52
71
 
53
72
  def server_middleware
54
- @server_chain ||= Middleware::Chain.new
73
+ @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
55
74
  yield @server_chain if block_given?
56
75
  @server_chain
57
76
  end
58
-
59
- def constantize(str)
60
- names = str.split("::")
61
- names.shift if names.empty? || names.first.empty?
62
-
63
- names.inject(Object) do |constant, name|
64
- constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
65
- end
66
- end
67
77
  end
68
78
  end
69
79
 
@@ -73,7 +83,7 @@ module Sidekiq
73
83
  class EmptyQueueError < RuntimeError; end
74
84
 
75
85
  module TestingClient
76
- def raw_push(payloads)
86
+ def atomic_push(conn, payloads)
77
87
  if Sidekiq::Testing.fake?
78
88
  payloads.each do |job|
79
89
  job = Sidekiq.load_json(Sidekiq.dump_json(job))
@@ -83,7 +93,7 @@ module Sidekiq
83
93
  true
84
94
  elsif Sidekiq::Testing.inline?
85
95
  payloads.each do |job|
86
- klass = Sidekiq::Testing.constantize(job["class"])
96
+ klass = Object.const_get(job["class"])
87
97
  job["id"] ||= SecureRandom.hex(12)
88
98
  job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
89
99
  klass.process_job(job_hash)
@@ -102,7 +112,7 @@ module Sidekiq
102
112
  # The Queues class is only for testing the fake queue implementation.
103
113
  # There are 2 data structures involved in tandem. This is due to the
104
114
  # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
105
- # to the array. Because the array was dervied from a filter of the total
115
+ # to the array. Because the array was derived from a filter of the total
106
116
  # jobs enqueued, it appeared as though the array didn't change.
107
117
  #
108
118
  # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
@@ -218,25 +228,9 @@ module Sidekiq
218
228
  # assert_equal 1, HardJob.jobs.size
219
229
  # assert_equal :something, HardJob.jobs[0]['args'][0]
220
230
  #
221
- # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
222
- # MyMailer.delay.send_welcome_email('foo@example.com')
223
- # assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
224
- #
225
231
  # You can also clear and drain all job types:
226
232
  #
227
- # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
228
- # assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
229
- #
230
- # MyMailer.delay.send_welcome_email('foo@example.com')
231
- # MyModel.delay.do_something_hard
232
- #
233
- # assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
234
- # assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
235
- #
236
- # Sidekiq::Worker.clear_all # or .drain_all
237
- #
238
- # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
239
- # assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
233
+ # Sidekiq::Job.clear_all # or .drain_all
240
234
  #
241
235
  # This can be useful to make sure jobs don't linger between tests:
242
236
  #
@@ -284,16 +278,16 @@ module Sidekiq
284
278
  def perform_one
285
279
  raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
286
280
  next_job = jobs.first
287
- Queues.delete_for(next_job["jid"], queue, to_s)
281
+ Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
288
282
  process_job(next_job)
289
283
  end
290
284
 
291
285
  def process_job(job)
292
- inst = new
293
- inst.jid = job["jid"]
294
- inst.bid = job["bid"] if inst.respond_to?(:bid=)
295
- Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
296
- execute_job(inst, job["args"])
286
+ instance = new
287
+ instance.jid = job["jid"]
288
+ instance.bid = job["bid"] if instance.respond_to?(:bid=)
289
+ Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
290
+ execute_job(instance, job["args"])
297
291
  end
298
292
  end
299
293
 
@@ -318,7 +312,7 @@ module Sidekiq
318
312
  job_classes = jobs.map { |job| job["class"] }.uniq
319
313
 
320
314
  job_classes.each do |job_class|
321
- Sidekiq::Testing.constantize(job_class).drain
315
+ Object.const_get(job_class).drain
322
316
  end
323
317
  end
324
318
  end
@@ -329,13 +323,10 @@ module Sidekiq
329
323
  def jobs_for(klass)
330
324
  jobs.select do |job|
331
325
  marshalled = job["args"][0]
332
- marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
326
+ marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
333
327
  end
334
328
  end
335
329
  end
336
-
337
- Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
338
- Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
339
330
  end
340
331
 
341
332
  if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
@@ -5,11 +5,18 @@ require "sidekiq/client"
5
5
 
6
6
  module Sidekiq
7
7
  class TransactionAwareClient
8
- def initialize(redis_pool)
9
- @redis_client = Client.new(redis_pool)
8
+ def initialize(pool: nil, config: nil)
9
+ @redis_client = Client.new(pool: pool, config: config)
10
+ end
11
+
12
+ def batching?
13
+ Thread.current[:sidekiq_batch]
10
14
  end
11
15
 
12
16
  def push(item)
17
+ # 6160 we can't support both Sidekiq::Batch and transactions.
18
+ return @redis_client.push(item) if batching?
19
+
13
20
  # pre-allocate the JID so we can return it immediately and
14
21
  # save it to the database as part of the transaction.
15
22
  item["jid"] ||= SecureRandom.hex(12)
@@ -34,11 +41,10 @@ module Sidekiq
34
41
  begin
35
42
  require "after_commit_everywhere"
36
43
  rescue LoadError
37
- Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
38
- raise
44
+ raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
39
45
  end
40
46
 
41
- default_job_options["client_class"] = Sidekiq::TransactionAwareClient
47
+ Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
42
48
  Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
43
49
  true
44
50
  end
@@ -1,5 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Sidekiq
4
- VERSION = "6.5.1"
4
+ VERSION = "7.3.6"
5
+ MAJOR = 7
6
+
7
+ def self.gem_version
8
+ Gem::Version.new(VERSION)
9
+ end
5
10
  end
@@ -15,11 +15,16 @@ module Sidekiq
15
15
  end
16
16
 
17
17
  def halt(res)
18
- throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
18
+ throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
19
19
  end
20
20
 
21
21
  def redirect(location)
22
- throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
22
+ throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
23
+ end
24
+
25
+ def reload_page
26
+ current_location = request.referer.gsub(request.base_url, "")
27
+ redirect current_location
23
28
  end
24
29
 
25
30
  def params
@@ -42,8 +47,13 @@ module Sidekiq
42
47
  def erb(content, options = {})
43
48
  if content.is_a? Symbol
44
49
  unless respond_to?(:"_erb_#{content}")
45
- src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
46
- WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
50
+ views = options[:views] || Web.settings.views
51
+ filename = "#{views}/#{content}.erb"
52
+ src = ERB.new(File.read(filename)).src
53
+
54
+ # Need to use lineno less by 1 because erb generates a
55
+ # comment before the source code.
56
+ WebAction.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
47
57
  def _erb_#{content}
48
58
  #{src}
49
59
  end
@@ -68,7 +78,7 @@ module Sidekiq
68
78
  end
69
79
 
70
80
  def json(payload)
71
- [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
81
+ [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
72
82
  end
73
83
 
74
84
  def initialize(env, block)
@@ -5,7 +5,7 @@ module Sidekiq
5
5
  extend WebRouter
6
6
 
7
7
  REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
8
- CSP_HEADER = [
8
+ CSP_HEADER_TEMPLATE = [
9
9
  "default-src 'self' https: http:",
10
10
  "child-src 'self'",
11
11
  "connect-src 'self' https: http: wss: ws:",
@@ -15,11 +15,17 @@ module Sidekiq
15
15
  "manifest-src 'self'",
16
16
  "media-src 'self'",
17
17
  "object-src 'none'",
18
- "script-src 'self' https: http: 'unsafe-inline'",
19
- "style-src 'self' https: http: 'unsafe-inline'",
18
+ "script-src 'self' 'nonce-!placeholder!'",
19
+ "style-src 'self' https: http: 'unsafe-inline'", # TODO Nonce in 8.0
20
20
  "worker-src 'self'",
21
21
  "base-uri 'self'"
22
22
  ].join("; ").freeze
23
+ METRICS_PERIODS = {
24
+ "1h" => 60,
25
+ "2h" => 120,
26
+ "4h" => 240,
27
+ "8h" => 480
28
+ }
23
29
 
24
30
  def initialize(klass)
25
31
  @klass = klass
@@ -43,9 +49,9 @@ module Sidekiq
43
49
 
44
50
  head "/" do
45
51
  # HEAD / is the cheapest heartbeat possible,
46
- # it hits Redis to ensure connectivity
47
- Sidekiq.redis { |c| c.llen("queue:default") }
48
- ""
52
+ # it hits Redis to ensure connectivity and returns
53
+ # the size of the default queue
54
+ Sidekiq.redis { |c| c.llen("queue:default") }.to_s
49
55
  end
50
56
 
51
57
  get "/" do
@@ -60,17 +66,46 @@ module Sidekiq
60
66
  erb(:dashboard)
61
67
  end
62
68
 
69
+ get "/metrics" do
70
+ x = params[:substr]
71
+ class_filter = (x.nil? || x == "") ? nil : Regexp.new(Regexp.escape(x), Regexp::IGNORECASE)
72
+
73
+ q = Sidekiq::Metrics::Query.new
74
+ @period = h((params[:period] || "")[0..1])
75
+ @periods = METRICS_PERIODS
76
+ minutes = @periods.fetch(@period, @periods.values.first)
77
+ @query_result = q.top_jobs(minutes: minutes, class_filter: class_filter)
78
+
79
+ erb(:metrics)
80
+ end
81
+
82
+ get "/metrics/:name" do
83
+ @name = route_params[:name]
84
+ @period = h((params[:period] || "")[0..1])
85
+ q = Sidekiq::Metrics::Query.new
86
+ @periods = METRICS_PERIODS
87
+ minutes = @periods.fetch(@period, @periods.values.first)
88
+ @query_result = q.for_job(@name, minutes: minutes)
89
+ erb(:metrics_for_job)
90
+ end
91
+
63
92
  get "/busy" do
93
+ @count = (params["count"] || 100).to_i
94
+ (@current_page, @total_size, @workset) = page_items(workset, params["page"], @count)
95
+
64
96
  erb(:busy)
65
97
  end
66
98
 
67
99
  post "/busy" do
68
100
  if params["identity"]
69
- p = Sidekiq::Process.new("identity" => params["identity"])
70
- p.quiet! if params["quiet"]
71
- p.stop! if params["stop"]
101
+ pro = Sidekiq::ProcessSet[params["identity"]]
102
+
103
+ pro.quiet! if params["quiet"]
104
+ pro.stop! if params["stop"]
72
105
  else
73
106
  processes.each do |pro|
107
+ next if pro.embedded?
108
+
74
109
  pro.quiet! if params["quiet"]
75
110
  pro.stop! if params["stop"]
76
111
  end
@@ -122,9 +157,15 @@ module Sidekiq
122
157
  end
123
158
 
124
159
  get "/morgue" do
125
- @count = (params["count"] || 25).to_i
126
- (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
127
- @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
160
+ x = params[:substr]
161
+
162
+ if x && x != ""
163
+ @dead = search(Sidekiq::DeadSet.new, x)
164
+ else
165
+ @count = (params["count"] || 25).to_i
166
+ (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
167
+ @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
168
+ end
128
169
 
129
170
  erb(:morgue)
130
171
  end
@@ -176,9 +217,15 @@ module Sidekiq
176
217
  end
177
218
 
178
219
  get "/retries" do
179
- @count = (params["count"] || 25).to_i
180
- (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
181
- @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
220
+ x = params[:substr]
221
+
222
+ if x && x != ""
223
+ @retries = search(Sidekiq::RetrySet.new, x)
224
+ else
225
+ @count = (params["count"] || 25).to_i
226
+ (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
227
+ @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
228
+ end
182
229
 
183
230
  erb(:retries)
184
231
  end
@@ -231,9 +278,15 @@ module Sidekiq
231
278
  end
232
279
 
233
280
  get "/scheduled" do
234
- @count = (params["count"] || 25).to_i
235
- (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
236
- @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
281
+ x = params[:substr]
282
+
283
+ if x && x != ""
284
+ @scheduled = search(Sidekiq::ScheduledSet.new, x)
285
+ else
286
+ @count = (params["count"] || 25).to_i
287
+ (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
288
+ @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
289
+ end
237
290
 
238
291
  erb(:scheduled)
239
292
  end
@@ -294,12 +347,24 @@ module Sidekiq
294
347
  end
295
348
 
296
349
  get "/stats/queues" do
297
- json Sidekiq::Stats::Queues.new.lengths
350
+ json Sidekiq::Stats.new.queues
351
+ end
352
+
353
+ post "/change_locale" do
354
+ locale = params["locale"]
355
+
356
+ match = available_locales.find { |available|
357
+ locale == available
358
+ }
359
+
360
+ session[:locale] = match if match
361
+
362
+ reload_page
298
363
  end
299
364
 
300
365
  def call(env)
301
366
  action = self.class.match(env)
302
- return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
367
+ return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action
303
368
 
304
369
  app = @klass
305
370
  resp = catch(:halt) do
@@ -316,16 +381,21 @@ module Sidekiq
316
381
  else
317
382
  # rendered content goes here
318
383
  headers = {
319
- "Content-Type" => "text/html",
320
- "Cache-Control" => "private, no-store",
321
- "Content-Language" => action.locale,
322
- "Content-Security-Policy" => CSP_HEADER
384
+ Rack::CONTENT_TYPE => "text/html",
385
+ Rack::CACHE_CONTROL => "private, no-store",
386
+ Web::CONTENT_LANGUAGE => action.locale,
387
+ Web::CONTENT_SECURITY_POLICY => process_csp(env, CSP_HEADER_TEMPLATE),
388
+ Web::X_CONTENT_TYPE_OPTIONS => "nosniff"
323
389
  }
324
390
  # we'll let Rack calculate Content-Length for us.
325
391
  [200, headers, [resp]]
326
392
  end
327
393
  end
328
394
 
395
+ def process_csp(env, input)
396
+ input.gsub("!placeholder!", env[:csp_nonce])
397
+ end
398
+
329
399
  def self.helpers(mod = nil, &block)
330
400
  if block
331
401
  WebAction.class_eval(&block)