sidekiq 8.0.2 → 8.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +108 -0
  3. data/README.md +15 -0
  4. data/bin/lint-herb +13 -0
  5. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +104 -58
  6. data/lib/generators/sidekiq/templates/job.rb.erb +1 -1
  7. data/lib/sidekiq/api.rb +30 -6
  8. data/lib/sidekiq/capsule.rb +4 -0
  9. data/lib/sidekiq/cli.rb +16 -4
  10. data/lib/sidekiq/client.rb +15 -1
  11. data/lib/sidekiq/component.rb +2 -1
  12. data/lib/sidekiq/config.rb +11 -6
  13. data/lib/sidekiq/fetch.rb +1 -0
  14. data/lib/sidekiq/job/iterable.rb +33 -14
  15. data/lib/sidekiq/job.rb +4 -2
  16. data/lib/sidekiq/job_logger.rb +5 -3
  17. data/lib/sidekiq/job_retry.rb +23 -8
  18. data/lib/sidekiq/launcher.rb +18 -9
  19. data/lib/sidekiq/loader.rb +57 -0
  20. data/lib/sidekiq/logger.rb +16 -9
  21. data/lib/sidekiq/metrics/tracking.rb +3 -0
  22. data/lib/sidekiq/middleware/current_attributes.rb +6 -2
  23. data/lib/sidekiq/middleware/i18n.rb +2 -0
  24. data/lib/sidekiq/monitor.rb +4 -8
  25. data/lib/sidekiq/profiler.rb +17 -3
  26. data/lib/sidekiq/rails.rb +46 -67
  27. data/lib/sidekiq/redis_connection.rb +2 -2
  28. data/lib/sidekiq/ring_buffer.rb +1 -0
  29. data/lib/sidekiq/scheduled.rb +7 -5
  30. data/lib/sidekiq/testing.rb +1 -1
  31. data/lib/sidekiq/transaction_aware_client.rb +13 -5
  32. data/lib/sidekiq/version.rb +1 -1
  33. data/lib/sidekiq/web/action.rb +45 -2
  34. data/lib/sidekiq/web/application.rb +22 -4
  35. data/lib/sidekiq/web/config.rb +3 -3
  36. data/lib/sidekiq/web/helpers.rb +26 -29
  37. data/lib/sidekiq/web.rb +23 -3
  38. data/lib/sidekiq.rb +5 -0
  39. data/sidekiq.gemspec +5 -5
  40. data/web/assets/images/logo.png +0 -0
  41. data/web/assets/images/status.png +0 -0
  42. data/web/assets/javascripts/application.js +36 -13
  43. data/web/assets/javascripts/dashboard.js +1 -1
  44. data/web/assets/stylesheets/style.css +30 -6
  45. data/web/locales/ar.yml +1 -0
  46. data/web/locales/cs.yml +1 -0
  47. data/web/locales/da.yml +1 -0
  48. data/web/locales/de.yml +1 -0
  49. data/web/locales/el.yml +1 -0
  50. data/web/locales/en.yml +1 -0
  51. data/web/locales/es.yml +1 -0
  52. data/web/locales/fa.yml +1 -0
  53. data/web/locales/fr.yml +2 -1
  54. data/web/locales/gd.yml +1 -0
  55. data/web/locales/he.yml +1 -0
  56. data/web/locales/hi.yml +1 -0
  57. data/web/locales/it.yml +8 -0
  58. data/web/locales/ja.yml +1 -0
  59. data/web/locales/ko.yml +1 -0
  60. data/web/locales/lt.yml +1 -0
  61. data/web/locales/nb.yml +1 -0
  62. data/web/locales/nl.yml +1 -0
  63. data/web/locales/pl.yml +1 -0
  64. data/web/locales/pt-BR.yml +1 -0
  65. data/web/locales/pt.yml +1 -0
  66. data/web/locales/ru.yml +1 -0
  67. data/web/locales/sv.yml +1 -0
  68. data/web/locales/ta.yml +1 -0
  69. data/web/locales/tr.yml +1 -0
  70. data/web/locales/uk.yml +6 -5
  71. data/web/locales/ur.yml +1 -0
  72. data/web/locales/vi.yml +1 -0
  73. data/web/locales/zh-CN.yml +1 -0
  74. data/web/locales/zh-TW.yml +1 -0
  75. data/web/views/{_footer.erb → _footer.html.erb} +1 -1
  76. data/web/views/{_metrics_period_select.erb → _metrics_period_select.html.erb} +1 -1
  77. data/web/views/{_paging.erb → _paging.html.erb} +0 -1
  78. data/web/views/_poll_link.html.erb +4 -0
  79. data/web/views/{busy.erb → busy.html.erb} +4 -8
  80. data/web/views/{dashboard.erb → dashboard.html.erb} +3 -3
  81. data/web/views/{dead.erb → dead.html.erb} +3 -3
  82. data/web/views/filtering.html.erb +6 -0
  83. data/web/views/{layout.erb → layout.html.erb} +8 -7
  84. data/web/views/{metrics.erb → metrics.html.erb} +9 -8
  85. data/web/views/{morgue.erb → morgue.html.erb} +8 -4
  86. data/web/views/{queue.erb → queue.html.erb} +2 -2
  87. data/web/views/{queues.erb → queues.html.erb} +4 -4
  88. data/web/views/{retries.erb → retries.html.erb} +9 -5
  89. data/web/views/{retry.erb → retry.html.erb} +2 -2
  90. data/web/views/{scheduled.erb → scheduled.html.erb} +9 -5
  91. data/web/views/{scheduled_job_info.erb → scheduled_job_info.html.erb} +2 -2
  92. metadata +37 -36
  93. data/lib/sidekiq/web/csrf_protection.rb +0 -183
  94. data/web/views/_poll_link.erb +0 -4
  95. data/web/views/filtering.erb +0 -6
  96. /data/web/views/{_job_info.erb → _job_info.html.erb} +0 -0
  97. /data/web/views/{_nav.erb → _nav.html.erb} +0 -0
  98. /data/web/views/{_summary.erb → _summary.html.erb} +0 -0
  99. /data/web/views/{metrics_for_job.erb → metrics_for_job.html.erb} +0 -0
  100. /data/web/views/{profiles.erb → profiles.html.erb} +0 -0
@@ -32,8 +32,14 @@ module Sidekiq
32
32
  @_runtime = 0
33
33
  @_args = nil
34
34
  @_cancelled = nil
35
+ @current_object = nil
35
36
  end
36
37
 
38
+ # Access to the current object while iterating.
39
+ # This value is not reset so the latest element is
40
+ # explicitly available to cleanup/complete callbacks.
41
+ attr_reader :current_object
42
+
37
43
  def arguments
38
44
  @_args
39
45
  end
@@ -49,7 +55,7 @@ module Sidekiq
49
55
  def cancel!
50
56
  return @_cancelled if cancelled?
51
57
 
52
- key = "it-#{jid}"
58
+ key = iteration_key
53
59
  _, result, _ = Sidekiq.redis do |c|
54
60
  c.pipelined do |p|
55
61
  p.hsetnx(key, "cancelled", Time.now.to_i)
@@ -137,7 +143,7 @@ module Sidekiq
137
143
  fetch_previous_iteration_state
138
144
 
139
145
  @_executions += 1
140
- @_start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
146
+ @_start_time = mono_now
141
147
 
142
148
  enumerator = build_enumerator(*args, cursor: @_cursor)
143
149
  unless enumerator
@@ -171,7 +177,7 @@ module Sidekiq
171
177
  private
172
178
 
173
179
  def is_cancelled?
174
- @_cancelled = Sidekiq.redis { |c| c.hget("it-#{jid}", "cancelled") }
180
+ @_cancelled = Sidekiq.redis { |c| c.hget(iteration_key, "cancelled") }
175
181
  end
176
182
 
177
183
  def fetch_previous_iteration_state
@@ -198,16 +204,17 @@ module Sidekiq
198
204
 
199
205
  time_limit = Sidekiq.default_configuration[:timeout]
200
206
  found_record = false
201
- state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
207
+ state_flushed_at = mono_now
202
208
 
203
209
  enumerator.each do |object, cursor|
204
210
  found_record = true
205
211
  @_cursor = cursor
212
+ @current_object = object
206
213
 
207
- is_interrupted = interrupted?
208
- if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
214
+ interrupt_job = interrupted? || should_interrupt?
215
+ if mono_now - state_flushed_at >= STATE_FLUSH_INTERVAL || interrupt_job
209
216
  _, _, cancelled = flush_state
210
- state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
217
+ state_flushed_at = mono_now
211
218
  if cancelled
212
219
  @_cancelled = true
213
220
  on_cancel
@@ -216,11 +223,14 @@ module Sidekiq
216
223
  end
217
224
  end
218
225
 
219
- return false if is_interrupted
226
+ return false if interrupt_job
220
227
 
221
- verify_iteration_time(time_limit, object) do
228
+ verify_iteration_time(time_limit) do
222
229
  around_iteration do
223
230
  each_iteration(object, *arguments)
231
+ rescue Exception
232
+ flush_state
233
+ raise
224
234
  end
225
235
  end
226
236
  end
@@ -228,16 +238,16 @@ module Sidekiq
228
238
  logger.debug("Enumerator found nothing to iterate!") unless found_record
229
239
  true
230
240
  ensure
231
- @_runtime += (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @_start_time)
241
+ @_runtime += (mono_now - @_start_time)
232
242
  end
233
243
 
234
- def verify_iteration_time(time_limit, object)
235
- start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
244
+ def verify_iteration_time(time_limit)
245
+ start = mono_now
236
246
  yield
237
- finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
247
+ finish = mono_now
238
248
  total = finish - start
239
249
  if total > time_limit
240
- logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d) when processing `%s`. This can lead to job processing problems during deploys" % [total, time_limit, object] }
250
+ logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d). This can lead to job processing problems during deploys" % [total, time_limit] }
241
251
  end
242
252
  end
243
253
 
@@ -263,6 +273,11 @@ module Sidekiq
263
273
  end
264
274
  end
265
275
 
276
+ def should_interrupt?
277
+ max_iteration_runtime = Sidekiq.default_configuration[:max_iteration_runtime]
278
+ max_iteration_runtime && (mono_now - @_start_time > max_iteration_runtime)
279
+ end
280
+
266
281
  def flush_state
267
282
  key = iteration_key
268
283
  state = {
@@ -298,6 +313,10 @@ module Sidekiq
298
313
  raise "Unexpected thrown value: #{completed.inspect}"
299
314
  end
300
315
  end
316
+
317
+ def mono_now
318
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
319
+ end
301
320
  end
302
321
  end
303
322
  end
data/lib/sidekiq/job.rb CHANGED
@@ -226,6 +226,8 @@ module Sidekiq
226
226
  end
227
227
  return nil unless result
228
228
 
229
+ verify_json(item)
230
+
229
231
  # round-trip the payload via JSON
230
232
  msg = Sidekiq.load_json(Sidekiq.dump_json(item))
231
233
 
@@ -248,9 +250,9 @@ module Sidekiq
248
250
  end
249
251
  alias_method :perform_sync, :perform_inline
250
252
 
251
- def perform_bulk(args, batch_size: 1_000)
253
+ def perform_bulk(args, **options)
252
254
  client = @klass.build_client
253
- client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
255
+ client.push_bulk(@opts.merge({"class" => @klass, "args" => args}, options))
254
256
  end
255
257
 
256
258
  # +interval+ must be a timestamp, numeric or something that acts
@@ -27,10 +27,12 @@ module Sidekiq
27
27
  # attribute to expose the underlying thing.
28
28
  h = {
29
29
  jid: job_hash["jid"],
30
- class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"]
30
+ class: job_hash["wrapped"] || job_hash["class"]
31
31
  }
32
- h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
33
- h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
32
+
33
+ @config[:logged_job_attributes].each do |attr|
34
+ h[attr.to_sym] = job_hash[attr] if job_hash.has_key?(attr)
35
+ end
34
36
 
35
37
  Thread.current[:sidekiq_context] = h
36
38
  level = job_hash["log_level"]
@@ -178,15 +178,21 @@ module Sidekiq
178
178
  msg["error_backtrace"] = compress_backtrace(lines)
179
179
  end
180
180
 
181
- return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
182
-
181
+ # retry_for and retry are mutually exclusive - if retry_for is set,
182
+ # we exclusively use duration-based retry logic and ignore count-based logic
183
183
  rf = msg["retry_for"]
184
- return retries_exhausted(jobinst, msg, exception) if rf && (time_for(msg["failed_at"]) + rf) < Time.now
184
+ if rf
185
+ return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
186
+ elsif count >= max_retry_attempts
187
+ return retries_exhausted(jobinst, msg, exception)
188
+ end
185
189
 
186
190
  strategy, delay = delay_for(jobinst, count, exception, msg)
187
191
  case strategy
188
192
  when :discard
189
- return # poof!
193
+ msg["discarded_at"] = now_ms
194
+
195
+ return run_death_handlers(msg, exception)
190
196
  when :kill
191
197
  return retries_exhausted(jobinst, msg, exception)
192
198
  end
@@ -255,13 +261,22 @@ module Sidekiq
255
261
  handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
256
262
  end
257
263
 
258
- return if rv == :discard # poof!
259
- send_to_morgue(msg) unless msg["dead"] == false
264
+ discarded = msg["dead"] == false || rv == :discard
265
+
266
+ if discarded
267
+ msg["discarded_at"] = now_ms
268
+ else
269
+ send_to_morgue(msg)
270
+ end
271
+
272
+ run_death_handlers(msg, exception)
273
+ end
260
274
 
275
+ def run_death_handlers(job, exception)
261
276
  @capsule.config.death_handlers.each do |handler|
262
- handler.call(msg, exception)
277
+ handler.call(job, exception)
263
278
  rescue => e
264
- handle_exception(e, {context: "Error calling death handler", job: msg})
279
+ handle_exception(e, {context: "Error calling death handler", job: job})
265
280
  end
266
281
  end
267
282
 
@@ -142,6 +142,12 @@ module Sidekiq
142
142
  key = identity
143
143
  fails = procd = 0
144
144
 
145
+ idle_timeout = config[:redis_idle_timeout]
146
+ if idle_timeout
147
+ config.capsules.each_value { |cap| cap.local_redis_pool.reap(idle_seconds: idle_timeout, &:close) }
148
+ config.local_redis_pool.reap(idle_seconds: idle_timeout, &:close)
149
+ end
150
+
145
151
  begin
146
152
  flush_stats
147
153
 
@@ -214,11 +220,11 @@ module Sidekiq
214
220
  # Log a warning if it's a disaster.
215
221
  if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
216
222
  logger.warn <<~EOM
217
- Your Redis network connection is performing extremely poorly.
223
+ Your Redis network connection appears to be performing poorly.
218
224
  Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
219
- Ensure Redis is running in the same AZ or datacenter as Sidekiq.
220
- If these values are close to 100,000, that means your Sidekiq process may be
221
- CPU-saturated; reduce your concurrency and/or see https://github.com/sidekiq/sidekiq/discussions/5039
225
+ Ensure Redis is running in the same AZ or datacenter as Sidekiq and that
226
+ your Sidekiq process is not CPU-saturated; reduce your concurrency and/or
227
+ see https://github.com/sidekiq/sidekiq/discussions/5039
222
228
  EOM
223
229
  RTT_READINGS.reset
224
230
  end
@@ -252,8 +258,15 @@ module Sidekiq
252
258
  "pid" => ::Process.pid,
253
259
  "tag" => @config[:tag] || "",
254
260
  "concurrency" => @config.total_concurrency,
261
+ "capsules" => @config.capsules.each_with_object({}) { |(name, cap), memo|
262
+ memo[name] = cap.to_h
263
+ },
264
+ #####
265
+ # TODO deprecated, remove in 9.0
266
+ # This data is now found in the `capsules` element above
255
267
  "queues" => @config.capsules.values.flat_map { |cap| cap.queues }.uniq,
256
- "weights" => to_weights,
268
+ "weights" => @config.capsules.values.map(&:weights),
269
+ #####
257
270
  "labels" => @config[:labels].to_a,
258
271
  "identity" => identity,
259
272
  "version" => Sidekiq::VERSION,
@@ -261,10 +274,6 @@ module Sidekiq
261
274
  }
262
275
  end
263
276
 
264
- def to_weights
265
- @config.capsules.values.map(&:weights)
266
- end
267
-
268
277
  def to_json
269
278
  # this data changes infrequently so dump it to a string
270
279
  # now so we don't need to dump it every heartbeat.
@@ -0,0 +1,57 @@
1
+ module Sidekiq
2
+ require "sidekiq/component"
3
+
4
+ class Loader
5
+ include Sidekiq::Component
6
+
7
+ def initialize(cfg = Sidekiq.default_configuration)
8
+ @config = cfg
9
+ @load_hooks = Hash.new { |h, k| h[k] = [] }
10
+ @loaded = Set.new
11
+ @lock = Mutex.new
12
+ end
13
+
14
+ # Declares a block that will be executed when a Sidekiq component is fully
15
+ # loaded. If the component has already loaded, the block is executed
16
+ # immediately.
17
+ #
18
+ # Sidekiq.loader.on_load(:api) do
19
+ # # extend the sidekiq API
20
+ # end
21
+ #
22
+ def on_load(name, &block)
23
+ # we don't want to hold the lock while calling the block
24
+ to_run = nil
25
+
26
+ @lock.synchronize do
27
+ if @loaded.include?(name)
28
+ to_run = block
29
+ else
30
+ @load_hooks[name] << block
31
+ end
32
+ end
33
+
34
+ to_run&.call
35
+ nil
36
+ end
37
+
38
+ # Executes all blocks registered to +name+ via on_load.
39
+ #
40
+ # Sidekiq.loader.run_load_hooks(:api)
41
+ #
42
+ # In the case of the above example, it will execute all hooks registered for +:api+.
43
+ #
44
+ def run_load_hooks(name)
45
+ hks = @lock.synchronize do
46
+ @loaded << name
47
+ @load_hooks.delete(name)
48
+ end
49
+
50
+ hks&.each do |blk|
51
+ blk.call
52
+ rescue => ex
53
+ handle_exception(ex, hook: name)
54
+ end
55
+ end
56
+ end
57
+ end
@@ -24,14 +24,15 @@ module Sidekiq
24
24
 
25
25
  class Logger < ::Logger
26
26
  module Formatters
27
- COLORS = {
28
- "DEBUG" => "\e[1;32mDEBUG\e[0m", # green
29
- "INFO" => "\e[1;34mINFO \e[0m", # blue
30
- "WARN" => "\e[1;33mWARN \e[0m", # yellow
31
- "ERROR" => "\e[1;31mERROR\e[0m", # red
32
- "FATAL" => "\e[1;35mFATAL\e[0m" # pink
33
- }
34
27
  class Base < ::Logger::Formatter
28
+ COLORS = {
29
+ "DEBUG" => "\e[1;32mDEBUG\e[0m", # green
30
+ "INFO" => "\e[1;34mINFO \e[0m", # blue
31
+ "WARN" => "\e[1;33mWARN \e[0m", # yellow
32
+ "ERROR" => "\e[1;31mERROR\e[0m", # red
33
+ "FATAL" => "\e[1;35mFATAL\e[0m" # pink
34
+ }
35
+
35
36
  def tid
36
37
  Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
37
38
  end
@@ -50,13 +51,19 @@ module Sidekiq
50
51
 
51
52
  class Pretty < Base
52
53
  def call(severity, time, program_name, message)
53
- "#{Formatters::COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
54
+ "#{COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
55
+ end
56
+ end
57
+
58
+ class Plain < Base
59
+ def call(severity, time, program_name, message)
60
+ "#{severity} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
54
61
  end
55
62
  end
56
63
 
57
64
  class WithoutTimestamp < Pretty
58
65
  def call(severity, time, program_name, message)
59
- "#{Formatters::COLORS[severity]} pid=#{::Process.pid} tid=#{tid} #{format_context}: #{message}\n"
66
+ "#{COLORS[severity]} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
60
67
  end
61
68
  end
62
69
 
@@ -147,4 +147,7 @@ Sidekiq.configure_server do |config|
147
147
  config.on(:beat) do
148
148
  exec.flush
149
149
  end
150
+ config.on(:exit) do
151
+ exec.flush
152
+ end
150
153
  end
@@ -50,7 +50,7 @@ module Sidekiq
50
50
  @cattrs = cattrs
51
51
  end
52
52
 
53
- def call(_, job, _, &block)
53
+ def call(_, job, *, &block)
54
54
  klass_attrs = {}
55
55
 
56
56
  @cattrs.each do |(key, strklass)|
@@ -71,11 +71,14 @@ module Sidekiq
71
71
  retried = false
72
72
 
73
73
  begin
74
+ set_succeeded = false
74
75
  klass.set(attrs) do
76
+ set_succeeded = true
75
77
  wrap(klass_attrs, &block)
76
78
  end
77
79
  rescue NoMethodError
78
- raise if retried
80
+ # Don't retry if the no method error didn't come from current attributes
81
+ raise if retried || set_succeeded
79
82
 
80
83
  # It is possible that the `CurrentAttributes` definition
81
84
  # was changed before the job started processing.
@@ -90,6 +93,7 @@ module Sidekiq
90
93
  def persist(klass_or_array, config = Sidekiq.default_configuration)
91
94
  cattrs = build_cattrs_hash(klass_or_array)
92
95
 
96
+ config.client_middleware.prepend Load, cattrs
93
97
  config.client_middleware.add Save, cattrs
94
98
  config.server_middleware.prepend Load, cattrs
95
99
  end
@@ -11,6 +11,7 @@ module Sidekiq::Middleware::I18n
11
11
  # to be sent to Sidekiq.
12
12
  class Client
13
13
  include Sidekiq::ClientMiddleware
14
+
14
15
  def call(_jobclass, job, _queue, _redis)
15
16
  job["locale"] ||= I18n.locale
16
17
  yield
@@ -20,6 +21,7 @@ module Sidekiq::Middleware::I18n
20
21
  # Pull the msg locale out and set the current thread to use it.
21
22
  class Server
22
23
  include Sidekiq::ServerMiddleware
24
+
23
25
  def call(_jobclass, job, _queue, &block)
24
26
  I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
25
27
  end
@@ -49,14 +49,10 @@ class Sidekiq::Monitor
49
49
  puts "---- Processes (#{process_set.size}) ----"
50
50
  process_set.each_with_index do |process, index|
51
51
  # Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
52
- #
53
- # Before:
54
- # ["default", "critical"]
55
- #
56
- # After:
57
- # {"default" => 1, "critical" => 10}
58
52
  queues =
59
- if process["weights"]
53
+ if process["capsules"] # 8.0.6+
54
+ process["capsules"].values.map { |x| x["weights"].keys.join(", ") }
55
+ elsif process["weights"]
60
56
  process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
61
57
  else
62
58
  process["queues"].sort
@@ -105,7 +101,7 @@ class Sidekiq::Monitor
105
101
  out << line
106
102
  line = " " * pad
107
103
  end
108
- line << value + ", "
104
+ line << value + "; "
109
105
  end
110
106
  out << line[0..-3]
111
107
  out.join("\n")
@@ -11,8 +11,10 @@ module Sidekiq
11
11
  }
12
12
 
13
13
  include Sidekiq::Component
14
+
14
15
  def initialize(config)
15
16
  @config = config
17
+ @vernier_output_dir = ENV.fetch("VERNIER_OUTPUT_DIR") { Dir.tmpdir }
16
18
  end
17
19
 
18
20
  def call(job, &block)
@@ -22,7 +24,6 @@ module Sidekiq
22
24
  type = job["class"]
23
25
  jid = job["jid"]
24
26
  started_at = Time.now
25
- options = DEFAULT_OPTIONS.merge((job["profiler_options"] || {}).transform_keys!(&:to_sym))
26
27
 
27
28
  rundata = {
28
29
  started_at: started_at.to_i,
@@ -30,13 +31,17 @@ module Sidekiq
30
31
  type: type,
31
32
  jid: jid,
32
33
  # .gz extension tells Vernier to compress the data
33
- filename: "#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
34
+ filename: File.join(
35
+ @vernier_output_dir,
36
+ "#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
37
+ )
34
38
  }
39
+ profiler_options = profiler_options(job, rundata)
35
40
 
36
41
  require "vernier"
37
42
  begin
38
43
  a = Time.now
39
- rc = Vernier.profile(**options.merge(out: rundata[:filename]), &block)
44
+ rc = Vernier.profile(**profiler_options, &block)
40
45
  b = Time.now
41
46
 
42
47
  # Failed jobs will raise an exception on previous line and skip this
@@ -55,5 +60,14 @@ module Sidekiq
55
60
  FileUtils.rm_f(rundata[:filename])
56
61
  end
57
62
  end
63
+
64
+ private
65
+
66
+ def profiler_options(job, rundata)
67
+ profiler_options = (job["profiler_options"] || {}).transform_keys(&:to_sym)
68
+ profiler_options[:mode] = profiler_options[:mode].to_sym if profiler_options[:mode]
69
+
70
+ DEFAULT_OPTIONS.merge(profiler_options, {out: rundata[:filename]})
71
+ end
58
72
  end
59
73
  end
data/lib/sidekiq/rails.rb CHANGED
@@ -1,84 +1,63 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "sidekiq/job"
4
- require "rails"
5
-
6
3
  module Sidekiq
7
- module ActiveJob
8
- # @api private
9
- class Wrapper
10
- include Sidekiq::Job
11
-
12
- def perform(job_data)
13
- ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
14
- end
15
- end
16
- end
17
-
18
- class Rails < ::Rails::Engine
19
- class Reloader
20
- def initialize(app = ::Rails.application)
21
- @app = app
22
- end
23
-
24
- def call
25
- params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
26
- @app.reloader.wrap(**params) do
27
- yield
4
+ begin
5
+ gem "railties", ">= 7.0"
6
+ require "rails"
7
+ require "sidekiq/job"
8
+ require_relative "../active_job/queue_adapters/sidekiq_adapter"
9
+
10
+ class Rails < ::Rails::Engine
11
+ class Reloader
12
+ def initialize(app = ::Rails.application)
13
+ @app = app
28
14
  end
29
- end
30
15
 
31
- def inspect
32
- "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
33
- end
16
+ def call
17
+ params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
18
+ @app.reloader.wrap(**params) do
19
+ yield
20
+ end
21
+ end
34
22
 
35
- def to_hash
36
- {app: @app.class.name}
37
- end
38
- end
23
+ def inspect
24
+ "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
25
+ end
39
26
 
40
- # By including the Options module, we allow AJs to directly control sidekiq features
41
- # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
42
- # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
43
- # manually retried, don't automatically die, etc.
44
- #
45
- # class SomeJob < ActiveJob::Base
46
- # queue_as :default
47
- # sidekiq_options retry: 3, backtrace: 10
48
- # def perform
49
- # end
50
- # end
51
- initializer "sidekiq.active_job_integration" do
52
- ActiveSupport.on_load(:active_job) do
53
- require_relative "../active_job/queue_adapters/sidekiq_adapter"
54
- include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
27
+ def to_hash
28
+ {app: @app.class.name}
29
+ end
55
30
  end
56
- end
57
31
 
58
- initializer "sidekiq.backtrace_cleaner" do
59
- Sidekiq.configure_server do |config|
60
- config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
32
+ initializer "sidekiq.backtrace_cleaner" do
33
+ Sidekiq.configure_server do |config|
34
+ config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
35
+ end
61
36
  end
62
- end
63
-
64
- # This hook happens after all initializers are run, just before returning
65
- # from config/environment.rb back to sidekiq/cli.rb.
66
- #
67
- # None of this matters on the client-side, only within the Sidekiq process itself.
68
- config.after_initialize do
69
- Sidekiq.configure_server do |config|
70
- config[:reloader] = Sidekiq::Rails::Reloader.new
71
37
 
72
- # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
73
- # it will appear in the Sidekiq console with all of the job context.
74
- unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
75
- if ::Rails.logger.respond_to?(:broadcast_to)
76
- ::Rails.logger.broadcast_to(config.logger)
77
- else
78
- ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
38
+ # This hook happens after all initializers are run, just before returning
39
+ # from config/environment.rb back to sidekiq/cli.rb.
40
+ #
41
+ # None of this matters on the client-side, only within the Sidekiq process itself.
42
+ config.after_initialize do
43
+ Sidekiq.configure_server do |config|
44
+ config[:reloader] = Sidekiq::Rails::Reloader.new
45
+
46
+ # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
47
+ # it will appear in the Sidekiq console with all of the job context.
48
+ unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
49
+ if ::Rails.logger.respond_to?(:broadcast_to)
50
+ ::Rails.logger.broadcast_to(config.logger)
51
+ elsif ::ActiveSupport::Logger.respond_to?(:broadcast)
52
+ ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
53
+ else
54
+ ::Rails.logger = ::ActiveSupport::BroadcastLogger.new(::Rails.logger, config.logger)
55
+ end
79
56
  end
80
57
  end
81
58
  end
82
59
  end
60
+ rescue Gem::LoadError
61
+ # Rails not available or version requirement not met
83
62
  end
84
63
  end
@@ -23,7 +23,7 @@ module Sidekiq
23
23
 
24
24
  size = symbolized_options.delete(:size) || 5
25
25
  pool_timeout = symbolized_options.delete(:pool_timeout) || 1
26
- pool_name = symbolized_options.delete(:pool_name)
26
+ symbolized_options.delete(:pool_name)
27
27
 
28
28
  # Default timeout in redis-client is 1 second, which can be too aggressive
29
29
  # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
@@ -33,7 +33,7 @@ module Sidekiq
33
33
  symbolized_options[:timeout] ||= 3
34
34
 
35
35
  redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
36
- ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
36
+ ConnectionPool.new(timeout: pool_timeout, size: size) do
37
37
  redis_config.new_client
38
38
  end
39
39
  end
@@ -6,6 +6,7 @@ module Sidekiq
6
6
  class RingBuffer
7
7
  include Enumerable
8
8
  extend Forwardable
9
+
9
10
  def_delegators :@buf, :[], :each, :size
10
11
 
11
12
  def initialize(size, default = 0)