sidekiq 6.4.1 → 6.5.12

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +112 -1
  3. data/README.md +1 -1
  4. data/bin/sidekiqload +18 -12
  5. data/lib/sidekiq/api.rb +222 -71
  6. data/lib/sidekiq/cli.rb +51 -37
  7. data/lib/sidekiq/client.rb +27 -28
  8. data/lib/sidekiq/component.rb +65 -0
  9. data/lib/sidekiq/delay.rb +1 -1
  10. data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
  11. data/lib/sidekiq/fetch.rb +18 -16
  12. data/lib/sidekiq/job_retry.rb +73 -52
  13. data/lib/sidekiq/job_util.rb +15 -9
  14. data/lib/sidekiq/launcher.rb +37 -33
  15. data/lib/sidekiq/logger.rb +5 -19
  16. data/lib/sidekiq/manager.rb +28 -25
  17. data/lib/sidekiq/metrics/deploy.rb +47 -0
  18. data/lib/sidekiq/metrics/query.rb +153 -0
  19. data/lib/sidekiq/metrics/shared.rb +94 -0
  20. data/lib/sidekiq/metrics/tracking.rb +134 -0
  21. data/lib/sidekiq/middleware/chain.rb +82 -38
  22. data/lib/sidekiq/middleware/current_attributes.rb +18 -12
  23. data/lib/sidekiq/middleware/i18n.rb +6 -4
  24. data/lib/sidekiq/middleware/modules.rb +21 -0
  25. data/lib/sidekiq/monitor.rb +2 -2
  26. data/lib/sidekiq/paginator.rb +11 -3
  27. data/lib/sidekiq/processor.rb +47 -41
  28. data/lib/sidekiq/rails.rb +19 -13
  29. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  30. data/lib/sidekiq/redis_connection.rb +80 -49
  31. data/lib/sidekiq/ring_buffer.rb +29 -0
  32. data/lib/sidekiq/scheduled.rb +53 -24
  33. data/lib/sidekiq/testing/inline.rb +4 -4
  34. data/lib/sidekiq/testing.rb +37 -36
  35. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  36. data/lib/sidekiq/version.rb +1 -1
  37. data/lib/sidekiq/web/action.rb +3 -3
  38. data/lib/sidekiq/web/application.rb +21 -5
  39. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  40. data/lib/sidekiq/web/helpers.rb +20 -7
  41. data/lib/sidekiq/web.rb +5 -1
  42. data/lib/sidekiq/worker.rb +24 -16
  43. data/lib/sidekiq.rb +106 -31
  44. data/sidekiq.gemspec +2 -2
  45. data/web/assets/javascripts/application.js +59 -26
  46. data/web/assets/javascripts/chart.min.js +13 -0
  47. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  48. data/web/assets/javascripts/dashboard.js +0 -17
  49. data/web/assets/javascripts/graph.js +16 -0
  50. data/web/assets/javascripts/metrics.js +262 -0
  51. data/web/assets/stylesheets/application.css +45 -3
  52. data/web/locales/el.yml +43 -19
  53. data/web/locales/en.yml +7 -0
  54. data/web/locales/ja.yml +7 -0
  55. data/web/locales/pt-br.yml +27 -9
  56. data/web/locales/zh-cn.yml +36 -11
  57. data/web/locales/zh-tw.yml +32 -7
  58. data/web/views/_nav.erb +1 -1
  59. data/web/views/_summary.erb +1 -1
  60. data/web/views/busy.erb +9 -4
  61. data/web/views/dashboard.erb +1 -0
  62. data/web/views/metrics.erb +69 -0
  63. data/web/views/metrics_for_job.erb +87 -0
  64. data/web/views/queue.erb +5 -1
  65. metadata +34 -9
  66. data/lib/sidekiq/exception_handler.rb +0 -27
  67. data/lib/sidekiq/util.rb +0 -108
@@ -1,10 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "sidekiq/scheduled"
4
- require "sidekiq/api"
5
-
6
3
  require "zlib"
7
4
  require "base64"
5
+ require "sidekiq/component"
8
6
 
9
7
  module Sidekiq
10
8
  ##
@@ -25,11 +23,11 @@ module Sidekiq
25
23
  #
26
24
  # A job looks like:
27
25
  #
28
- # { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
26
+ # { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
29
27
  #
30
28
  # The 'retry' option also accepts a number (in place of 'true'):
31
29
  #
32
- # { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
30
+ # { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
33
31
  #
34
32
  # The job will be retried this number of times before giving up. (If simply
35
33
  # 'true', Sidekiq retries 25 times)
@@ -53,11 +51,11 @@ module Sidekiq
53
51
  #
54
52
  # Sidekiq.options[:max_retries] = 7
55
53
  #
56
- # or limit the number of retries for a particular worker and send retries to
54
+ # or limit the number of retries for a particular job and send retries to
57
55
  # a low priority queue with:
58
56
  #
59
- # class MyWorker
60
- # include Sidekiq::Worker
57
+ # class MyJob
58
+ # include Sidekiq::Job
61
59
  # sidekiq_options retry: 10, retry_queue: 'low'
62
60
  # end
63
61
  #
@@ -66,17 +64,18 @@ module Sidekiq
66
64
 
67
65
  class Skip < Handled; end
68
66
 
69
- include Sidekiq::Util
67
+ include Sidekiq::Component
70
68
 
71
69
  DEFAULT_MAX_RETRY_ATTEMPTS = 25
72
70
 
73
- def initialize(options = {})
74
- @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
71
+ def initialize(options)
72
+ @config = options
73
+ @max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
75
74
  end
76
75
 
77
76
  # The global retry handler requires only the barest of data.
78
77
  # We want to be able to retry as much as possible so we don't
79
- # require the worker to be instantiated.
78
+ # require the job to be instantiated.
80
79
  def global(jobstr, queue)
81
80
  yield
82
81
  rescue Handled => ex
@@ -90,7 +89,7 @@ module Sidekiq
90
89
 
91
90
  msg = Sidekiq.load_json(jobstr)
92
91
  if msg["retry"]
93
- attempt_retry(nil, msg, queue, e)
92
+ process_retry(nil, msg, queue, e)
94
93
  else
95
94
  Sidekiq.death_handlers.each do |handler|
96
95
  handler.call(msg, e)
@@ -103,14 +102,14 @@ module Sidekiq
103
102
  end
104
103
 
105
104
  # The local retry support means that any errors that occur within
106
- # this block can be associated with the given worker instance.
105
+ # this block can be associated with the given job instance.
107
106
  # This is required to support the `sidekiq_retries_exhausted` block.
108
107
  #
109
108
  # Note that any exception from the block is wrapped in the Skip
110
109
  # exception so the global block does not reprocess the error. The
111
110
  # Skip exception is unwrapped within Sidekiq::Processor#process before
112
111
  # calling the handle_exception handlers.
113
- def local(worker, jobstr, queue)
112
+ def local(jobinst, jobstr, queue)
114
113
  yield
115
114
  rescue Handled => ex
116
115
  raise ex
@@ -123,11 +122,11 @@ module Sidekiq
123
122
 
124
123
  msg = Sidekiq.load_json(jobstr)
125
124
  if msg["retry"].nil?
126
- msg["retry"] = worker.class.get_sidekiq_options["retry"]
125
+ msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
127
126
  end
128
127
 
129
128
  raise e unless msg["retry"]
130
- attempt_retry(worker, msg, queue, e)
129
+ process_retry(jobinst, msg, queue, e)
131
130
  # We've handled this error associated with this job, don't
132
131
  # need to handle it at the global level
133
132
  raise Skip
@@ -135,10 +134,10 @@ module Sidekiq
135
134
 
136
135
  private
137
136
 
138
- # Note that +worker+ can be nil here if an error is raised before we can
139
- # instantiate the worker instance. All access must be guarded and
137
+ # Note that +jobinst+ can be nil here if an error is raised before we can
138
+ # instantiate the job instance. All access must be guarded and
140
139
  # best effort.
141
- def attempt_retry(worker, msg, queue, exception)
140
+ def process_retry(jobinst, msg, queue, exception)
142
141
  max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
143
142
 
144
143
  msg["queue"] = (msg["retry_queue"] || queue)
@@ -169,24 +168,54 @@ module Sidekiq
169
168
  msg["error_backtrace"] = compress_backtrace(lines)
170
169
  end
171
170
 
172
- if count < max_retry_attempts
173
- delay = delay_for(worker, count, exception)
174
- # Logging here can break retries if the logging device raises ENOSPC #3979
175
- # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
176
- retry_at = Time.now.to_f + delay
177
- payload = Sidekiq.dump_json(msg)
178
- Sidekiq.redis do |conn|
179
- conn.zadd("retry", retry_at.to_s, payload)
180
- end
181
- else
182
- # Goodbye dear message, you (re)tried your best I'm sure.
183
- retries_exhausted(worker, msg, exception)
171
+ # Goodbye dear message, you (re)tried your best I'm sure.
172
+ return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
173
+
174
+ strategy, delay = delay_for(jobinst, count, exception)
175
+ case strategy
176
+ when :discard
177
+ return # poof!
178
+ when :kill
179
+ return retries_exhausted(jobinst, msg, exception)
180
+ end
181
+
182
+ # Logging here can break retries if the logging device raises ENOSPC #3979
183
+ # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
184
+ jitter = rand(10) * (count + 1)
185
+ retry_at = Time.now.to_f + delay + jitter
186
+ payload = Sidekiq.dump_json(msg)
187
+ redis do |conn|
188
+ conn.zadd("retry", retry_at.to_s, payload)
189
+ end
190
+ end
191
+
192
+ # returns (strategy, seconds)
193
+ def delay_for(jobinst, count, exception)
194
+ rv = begin
195
+ # sidekiq_retry_in can return two different things:
196
+ # 1. When to retry next, as an integer of seconds
197
+ # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
198
+ jobinst&.sidekiq_retry_in_block&.call(count, exception)
199
+ rescue Exception => e
200
+ handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
201
+ nil
202
+ end
203
+
204
+ delay = (count**4) + 15
205
+ if Integer === rv && rv > 0
206
+ delay = rv
207
+ elsif rv == :discard
208
+ return [:discard, nil] # do nothing, job goes poof
209
+ elsif rv == :kill
210
+ return [:kill, nil]
184
211
  end
212
+
213
+ [:default, delay]
185
214
  end
186
215
 
187
- def retries_exhausted(worker, msg, exception)
216
+ def retries_exhausted(jobinst, msg, exception)
188
217
  begin
189
- block = worker&.sidekiq_retries_exhausted_block
218
+ block = jobinst&.sidekiq_retries_exhausted_block
190
219
  block&.call(msg, exception)
191
220
  rescue => e
192
221
  handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
@@ -194,7 +223,7 @@ module Sidekiq
194
223
 
195
224
  send_to_morgue(msg) unless msg["dead"] == false
196
225
 
197
- Sidekiq.death_handlers.each do |handler|
226
+ config.death_handlers.each do |handler|
198
227
  handler.call(msg, exception)
199
228
  rescue => e
200
229
  handle_exception(e, {context: "Error calling death handler", job: msg})
@@ -204,7 +233,15 @@ module Sidekiq
204
233
  def send_to_morgue(msg)
205
234
  logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
206
235
  payload = Sidekiq.dump_json(msg)
207
- DeadSet.new.kill(payload, notify_failure: false)
236
+ now = Time.now.to_f
237
+
238
+ config.redis do |conn|
239
+ conn.multi do |xa|
240
+ xa.zadd("dead", now.to_s, payload)
241
+ xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
242
+ xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
243
+ end
244
+ end
208
245
  end
209
246
 
210
247
  def retry_attempts_from(msg_retry, default)
@@ -215,22 +252,6 @@ module Sidekiq
215
252
  end
216
253
  end
217
254
 
218
- def delay_for(worker, count, exception)
219
- jitter = rand(10) * (count + 1)
220
- if worker&.sidekiq_retry_in_block
221
- custom_retry_in = retry_in(worker, count, exception).to_i
222
- return custom_retry_in + jitter if custom_retry_in > 0
223
- end
224
- (count**4) + 15 + jitter
225
- end
226
-
227
- def retry_in(worker, count, exception)
228
- worker.sidekiq_retry_in_block.call(count, exception)
229
- rescue Exception => e
230
- handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
231
- nil
232
- end
233
-
234
255
  def exception_caused_by_shutdown?(e, checked_causes = [])
235
256
  return false unless e.cause
236
257
 
@@ -4,7 +4,8 @@ require "time"
4
4
  module Sidekiq
5
5
  module JobUtil
6
6
  # These functions encapsulate various job utilities.
7
- # They must be simple and free from side effects.
7
+
8
+ TRANSIENT_ATTRIBUTES = %w[]
8
9
 
9
10
  def validate(item)
10
11
  raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
@@ -12,16 +13,19 @@ module Sidekiq
12
13
  raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
13
14
  raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
14
15
  raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
16
+ end
15
17
 
16
- if Sidekiq.options[:on_complex_arguments] == :raise
18
+ def verify_json(item)
19
+ job_class = item["wrapped"] || item["class"]
20
+ if Sidekiq[:on_complex_arguments] == :raise
17
21
  msg = <<~EOM
18
- Job arguments to #{item["class"]} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
22
+ Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
19
23
  To disable this error, remove `Sidekiq.strict_args!` from your initializer.
20
24
  EOM
21
25
  raise(ArgumentError, msg) unless json_safe?(item)
22
- elsif Sidekiq.options[:on_complex_arguments] == :warn
26
+ elsif Sidekiq[:on_complex_arguments] == :warn
23
27
  Sidekiq.logger.warn <<~EOM unless json_safe?(item)
24
- Job arguments to #{item["class"]} do not serialize to JSON safely. This will raise an error in
28
+ Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
25
29
  Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
26
30
  by calling `Sidekiq.strict_args!` during Sidekiq initialization.
27
31
  EOM
@@ -39,20 +43,22 @@ module Sidekiq
39
43
 
40
44
  raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
41
45
 
46
+ # remove job attributes which aren't necessary to persist into Redis
47
+ TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
48
+
49
+ item["jid"] ||= SecureRandom.hex(12)
42
50
  item["class"] = item["class"].to_s
43
51
  item["queue"] = item["queue"].to_s
44
- item["jid"] ||= SecureRandom.hex(12)
45
52
  item["created_at"] ||= Time.now.to_f
46
-
47
53
  item
48
54
  end
49
55
 
50
56
  def normalized_hash(item_class)
51
57
  if item_class.is_a?(Class)
52
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
58
+ raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
53
59
  item_class.get_sidekiq_options
54
60
  else
55
- Sidekiq.default_worker_options
61
+ Sidekiq.default_job_options
56
62
  end
57
63
  end
58
64
 
@@ -3,11 +3,12 @@
3
3
  require "sidekiq/manager"
4
4
  require "sidekiq/fetch"
5
5
  require "sidekiq/scheduled"
6
+ require "sidekiq/ring_buffer"
6
7
 
7
8
  module Sidekiq
8
9
  # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
9
10
  class Launcher
10
- include Util
11
+ include Sidekiq::Component
11
12
 
12
13
  STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
13
14
 
@@ -15,18 +16,18 @@ module Sidekiq
15
16
  proc { "sidekiq" },
16
17
  proc { Sidekiq::VERSION },
17
18
  proc { |me, data| data["tag"] },
18
- proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
19
+ proc { |me, data| "[#{Processor::WORK_STATE.size} of #{data["concurrency"]} busy]" },
19
20
  proc { |me, data| "stopping" if me.stopping? }
20
21
  ]
21
22
 
22
23
  attr_accessor :manager, :poller, :fetcher
23
24
 
24
25
  def initialize(options)
26
+ @config = options
25
27
  options[:fetch] ||= BasicFetch.new(options)
26
28
  @manager = Sidekiq::Manager.new(options)
27
- @poller = Sidekiq::Scheduled::Poller.new
29
+ @poller = Sidekiq::Scheduled::Poller.new(options)
28
30
  @done = false
29
- @options = options
30
31
  end
31
32
 
32
33
  def run
@@ -43,11 +44,9 @@ module Sidekiq
43
44
  @poller.terminate
44
45
  end
45
46
 
46
- # Shuts down the process. This method does not
47
- # return until all work is complete and cleaned up.
48
- # It can take up to the timeout to complete.
47
+ # Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
49
48
  def stop
50
- deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
49
+ deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
51
50
 
52
51
  @done = true
53
52
  @manager.quiet
@@ -55,10 +54,10 @@ module Sidekiq
55
54
 
56
55
  @manager.stop(deadline)
57
56
 
58
- # Requeue everything in case there was a worker who grabbed work while stopped
57
+ # Requeue everything in case there was a thread which fetched a job while the process was stopped.
59
58
  # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
60
- strategy = @options[:fetch]
61
- strategy.bulk_requeue([], @options)
59
+ strategy = @config[:fetch]
60
+ strategy.bulk_requeue([], @config)
62
61
 
63
62
  clear_heartbeat
64
63
  end
@@ -76,17 +75,19 @@ module Sidekiq
76
75
  heartbeat
77
76
  sleep BEAT_PAUSE
78
77
  end
79
- Sidekiq.logger.info("Heartbeat stopping...")
78
+ logger.info("Heartbeat stopping...")
80
79
  end
81
80
 
82
81
  def clear_heartbeat
82
+ flush_stats
83
+
83
84
  # Remove record from Redis since we are shutting down.
84
85
  # Note we don't stop the heartbeat thread; if the process
85
86
  # doesn't actually exit, it'll reappear in the Web UI.
86
- Sidekiq.redis do |conn|
87
+ redis do |conn|
87
88
  conn.pipelined do |pipeline|
88
- pipeline.srem("processes", identity)
89
- pipeline.unlink("#{identity}:workers")
89
+ pipeline.srem("processes", [identity])
90
+ pipeline.unlink("#{identity}:work")
90
91
  end
91
92
  end
92
93
  rescue
@@ -99,7 +100,7 @@ module Sidekiq
99
100
 
100
101
  end
101
102
 
102
- def self.flush_stats
103
+ def flush_stats
103
104
  fails = Processor::FAILURE.reset
104
105
  procd = Processor::PROCESSED.reset
105
106
  return if fails + procd == 0
@@ -123,7 +124,6 @@ module Sidekiq
123
124
  Sidekiq.logger.warn("Unable to flush stats: #{ex}")
124
125
  end
125
126
  end
126
- at_exit(&method(:flush_stats))
127
127
 
128
128
  def ❤
129
129
  key = identity
@@ -132,12 +132,11 @@ module Sidekiq
132
132
  begin
133
133
  fails = Processor::FAILURE.reset
134
134
  procd = Processor::PROCESSED.reset
135
- curstate = Processor::WORKER_STATE.dup
135
+ curstate = Processor::WORK_STATE.dup
136
136
 
137
- workers_key = "#{key}:workers"
138
137
  nowdate = Time.now.utc.strftime("%Y-%m-%d")
139
138
 
140
- Sidekiq.redis do |conn|
139
+ redis do |conn|
141
140
  conn.multi do |transaction|
142
141
  transaction.incrby("stat:processed", procd)
143
142
  transaction.incrby("stat:processed:#{nowdate}", procd)
@@ -146,12 +145,16 @@ module Sidekiq
146
145
  transaction.incrby("stat:failed", fails)
147
146
  transaction.incrby("stat:failed:#{nowdate}", fails)
148
147
  transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
148
+ end
149
149
 
150
- transaction.unlink(workers_key)
150
+ # work is the current set of executing jobs
151
+ work_key = "#{key}:work"
152
+ conn.pipelined do |transaction|
153
+ transaction.unlink(work_key)
151
154
  curstate.each_pair do |tid, hash|
152
- transaction.hset(workers_key, tid, Sidekiq.dump_json(hash))
155
+ transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
153
156
  end
154
- transaction.expire(workers_key, 60)
157
+ transaction.expire(work_key, 60)
155
158
  end
156
159
  end
157
160
 
@@ -160,15 +163,15 @@ module Sidekiq
160
163
  fails = procd = 0
161
164
  kb = memory_usage(::Process.pid)
162
165
 
163
- _, exists, _, _, msg = Sidekiq.redis { |conn|
166
+ _, exists, _, _, msg = redis { |conn|
164
167
  conn.multi { |transaction|
165
- transaction.sadd("processes", key)
168
+ transaction.sadd("processes", [key])
166
169
  transaction.exists?(key)
167
170
  transaction.hmset(key, "info", to_json,
168
171
  "busy", curstate.size,
169
172
  "beat", Time.now.to_f,
170
173
  "rtt_us", rtt,
171
- "quiet", @done,
174
+ "quiet", @done.to_s,
172
175
  "rss", kb)
173
176
  transaction.expire(key, 60)
174
177
  transaction.rpop("#{key}-signals")
@@ -177,6 +180,7 @@ module Sidekiq
177
180
 
178
181
  # first heartbeat or recovering from an outage and need to reestablish our heartbeat
179
182
  fire_event(:heartbeat) unless exists
183
+ fire_event(:beat, oneshot: false)
180
184
 
181
185
  return unless msg
182
186
 
@@ -198,7 +202,7 @@ module Sidekiq
198
202
 
199
203
  def check_rtt
200
204
  a = b = 0
201
- Sidekiq.redis do |x|
205
+ redis do |x|
202
206
  a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
203
207
  x.ping
204
208
  b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
@@ -209,12 +213,12 @@ module Sidekiq
209
213
  # Workable is < 10,000µs
210
214
  # Log a warning if it's a disaster.
211
215
  if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
212
- Sidekiq.logger.warn <<~EOM
216
+ logger.warn <<~EOM
213
217
  Your Redis network connection is performing extremely poorly.
214
218
  Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
215
219
  Ensure Redis is running in the same AZ or datacenter as Sidekiq.
216
220
  If these values are close to 100,000, that means your Sidekiq process may be
217
- CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
221
+ CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
218
222
  EOM
219
223
  RTT_READINGS.reset
220
224
  end
@@ -246,10 +250,10 @@ module Sidekiq
246
250
  "hostname" => hostname,
247
251
  "started_at" => Time.now.to_f,
248
252
  "pid" => ::Process.pid,
249
- "tag" => @options[:tag] || "",
250
- "concurrency" => @options[:concurrency],
251
- "queues" => @options[:queues].uniq,
252
- "labels" => @options[:labels],
253
+ "tag" => @config[:tag] || "",
254
+ "concurrency" => @config[:concurrency],
255
+ "queues" => @config[:queues].uniq,
256
+ "labels" => @config[:labels],
253
257
  "identity" => identity
254
258
  }
255
259
  end
@@ -18,7 +18,7 @@ module Sidekiq
18
18
  end
19
19
 
20
20
  def self.add(k, v)
21
- Thread.current[:sidekiq_context][k] = v
21
+ current[k] = v
22
22
  end
23
23
  end
24
24
 
@@ -35,24 +35,10 @@ module Sidekiq
35
35
  nil
36
36
  end
37
37
 
38
- def debug?
39
- level <= 0
40
- end
41
-
42
- def info?
43
- level <= 1
44
- end
45
-
46
- def warn?
47
- level <= 2
48
- end
49
-
50
- def error?
51
- level <= 3
52
- end
53
-
54
- def fatal?
55
- level <= 4
38
+ LEVELS.each do |level, numeric_level|
39
+ define_method("#{level}?") do
40
+ local_level.nil? ? super() : local_level <= numeric_level
41
+ end
56
42
  end
57
43
 
58
44
  def local_level
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "sidekiq/util"
4
3
  require "sidekiq/processor"
5
4
  require "sidekiq/fetch"
6
5
  require "set"
@@ -21,37 +20,34 @@ module Sidekiq
21
20
  # the shutdown process. The other tasks are performed by other threads.
22
21
  #
23
22
  class Manager
24
- include Util
23
+ include Sidekiq::Component
25
24
 
26
25
  attr_reader :workers
27
- attr_reader :options
28
26
 
29
27
  def initialize(options = {})
28
+ @config = options
30
29
  logger.debug { options.inspect }
31
- @options = options
32
30
  @count = options[:concurrency] || 10
33
31
  raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
34
32
 
35
33
  @done = false
36
34
  @workers = Set.new
37
35
  @count.times do
38
- @workers << Processor.new(self, options)
36
+ @workers << Processor.new(@config, &method(:processor_result))
39
37
  end
40
38
  @plock = Mutex.new
41
39
  end
42
40
 
43
41
  def start
44
- @workers.each do |x|
45
- x.start
46
- end
42
+ @workers.each(&:start)
47
43
  end
48
44
 
49
45
  def quiet
50
46
  return if @done
51
47
  @done = true
52
48
 
53
- logger.info { "Terminating quiet workers" }
54
- @workers.each { |x| x.terminate }
49
+ logger.info { "Terminating quiet threads" }
50
+ @workers.each(&:terminate)
55
51
  fire_event(:quiet, reverse: true)
56
52
  end
57
53
 
@@ -65,24 +61,18 @@ module Sidekiq
65
61
  sleep PAUSE_TIME
66
62
  return if @workers.empty?
67
63
 
68
- logger.info { "Pausing to allow workers to finish..." }
64
+ logger.info { "Pausing to allow jobs to finish..." }
69
65
  wait_for(deadline) { @workers.empty? }
70
66
  return if @workers.empty?
71
67
 
72
68
  hard_shutdown
73
69
  end
74
70
 
75
- def processor_stopped(processor)
76
- @plock.synchronize do
77
- @workers.delete(processor)
78
- end
79
- end
80
-
81
- def processor_died(processor, reason)
71
+ def processor_result(processor, reason = nil)
82
72
  @plock.synchronize do
83
73
  @workers.delete(processor)
84
74
  unless @done
85
- p = Processor.new(self, options)
75
+ p = Processor.new(@config, &method(:processor_result))
86
76
  @workers << p
87
77
  p.start
88
78
  end
@@ -96,7 +86,7 @@ module Sidekiq
96
86
  private
97
87
 
98
88
  def hard_shutdown
99
- # We've reached the timeout and we still have busy workers.
89
+ # We've reached the timeout and we still have busy threads.
100
90
  # They must die but their jobs shall live on.
101
91
  cleanup = nil
102
92
  @plock.synchronize do
@@ -106,17 +96,17 @@ module Sidekiq
106
96
  if cleanup.size > 0
107
97
  jobs = cleanup.map { |p| p.job }.compact
108
98
 
109
- logger.warn { "Terminating #{cleanup.size} busy worker threads" }
110
- logger.warn { "Work still in progress #{jobs.inspect}" }
99
+ logger.warn { "Terminating #{cleanup.size} busy threads" }
100
+ logger.debug { "Jobs still in progress #{jobs.inspect}" }
111
101
 
112
102
  # Re-enqueue unfinished jobs
113
103
  # NOTE: You may notice that we may push a job back to redis before
114
- # the worker thread is terminated. This is ok because Sidekiq's
104
+ # the thread is terminated. This is ok because Sidekiq's
115
105
  # contract says that jobs are run AT LEAST once. Process termination
116
106
  # is delayed until we're certain the jobs are back in Redis because
117
107
  # it is worse to lose a job than to run it twice.
118
- strategy = @options[:fetch]
119
- strategy.bulk_requeue(jobs, @options)
108
+ strategy = @config[:fetch]
109
+ strategy.bulk_requeue(jobs, @config)
120
110
  end
121
111
 
122
112
  cleanup.each do |processor|
@@ -129,5 +119,18 @@ module Sidekiq
129
119
  deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
130
120
  wait_for(deadline) { @workers.empty? }
131
121
  end
122
+
123
+ # hack for quicker development / testing environment #2774
124
+ PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
125
+
126
+ # Wait for the orblock to be true or the deadline passed.
127
+ def wait_for(deadline, &condblock)
128
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
129
+ while remaining > PAUSE_TIME
130
+ return if condblock.call
131
+ sleep PAUSE_TIME
132
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
133
+ end
134
+ end
132
135
  end
133
136
  end