sidekiq 6.4.0 → 6.5.12

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +119 -1
  3. data/README.md +6 -1
  4. data/bin/sidekiq +3 -3
  5. data/bin/sidekiqload +70 -66
  6. data/bin/sidekiqmon +1 -1
  7. data/lib/sidekiq/api.rb +255 -100
  8. data/lib/sidekiq/cli.rb +60 -38
  9. data/lib/sidekiq/client.rb +44 -30
  10. data/lib/sidekiq/component.rb +65 -0
  11. data/lib/sidekiq/delay.rb +2 -2
  12. data/lib/sidekiq/extensions/action_mailer.rb +2 -2
  13. data/lib/sidekiq/extensions/active_record.rb +2 -2
  14. data/lib/sidekiq/extensions/class_methods.rb +2 -2
  15. data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
  16. data/lib/sidekiq/fetch.rb +20 -18
  17. data/lib/sidekiq/job_logger.rb +15 -27
  18. data/lib/sidekiq/job_retry.rb +73 -52
  19. data/lib/sidekiq/job_util.rb +15 -9
  20. data/lib/sidekiq/launcher.rb +58 -54
  21. data/lib/sidekiq/logger.rb +8 -18
  22. data/lib/sidekiq/manager.rb +28 -25
  23. data/lib/sidekiq/metrics/deploy.rb +47 -0
  24. data/lib/sidekiq/metrics/query.rb +153 -0
  25. data/lib/sidekiq/metrics/shared.rb +94 -0
  26. data/lib/sidekiq/metrics/tracking.rb +134 -0
  27. data/lib/sidekiq/middleware/chain.rb +82 -38
  28. data/lib/sidekiq/middleware/current_attributes.rb +18 -12
  29. data/lib/sidekiq/middleware/i18n.rb +6 -4
  30. data/lib/sidekiq/middleware/modules.rb +21 -0
  31. data/lib/sidekiq/monitor.rb +2 -2
  32. data/lib/sidekiq/paginator.rb +17 -9
  33. data/lib/sidekiq/processor.rb +47 -41
  34. data/lib/sidekiq/rails.rb +19 -13
  35. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  36. data/lib/sidekiq/redis_connection.rb +80 -49
  37. data/lib/sidekiq/ring_buffer.rb +29 -0
  38. data/lib/sidekiq/scheduled.rb +53 -24
  39. data/lib/sidekiq/testing/inline.rb +4 -4
  40. data/lib/sidekiq/testing.rb +37 -36
  41. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  42. data/lib/sidekiq/version.rb +1 -1
  43. data/lib/sidekiq/web/action.rb +3 -3
  44. data/lib/sidekiq/web/application.rb +21 -5
  45. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  46. data/lib/sidekiq/web/helpers.rb +21 -8
  47. data/lib/sidekiq/web.rb +8 -4
  48. data/lib/sidekiq/worker.rb +26 -20
  49. data/lib/sidekiq.rb +107 -31
  50. data/sidekiq.gemspec +2 -2
  51. data/web/assets/javascripts/application.js +59 -26
  52. data/web/assets/javascripts/chart.min.js +13 -0
  53. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  54. data/web/assets/javascripts/dashboard.js +0 -17
  55. data/web/assets/javascripts/graph.js +16 -0
  56. data/web/assets/javascripts/metrics.js +262 -0
  57. data/web/assets/stylesheets/application.css +45 -1
  58. data/web/locales/el.yml +43 -19
  59. data/web/locales/en.yml +7 -0
  60. data/web/locales/ja.yml +7 -0
  61. data/web/locales/pt-br.yml +27 -9
  62. data/web/locales/zh-cn.yml +36 -11
  63. data/web/locales/zh-tw.yml +32 -7
  64. data/web/views/_nav.erb +1 -1
  65. data/web/views/_summary.erb +1 -1
  66. data/web/views/busy.erb +9 -4
  67. data/web/views/dashboard.erb +1 -0
  68. data/web/views/metrics.erb +69 -0
  69. data/web/views/metrics_for_job.erb +87 -0
  70. data/web/views/queue.erb +5 -1
  71. metadata +34 -9
  72. data/lib/sidekiq/exception_handler.rb +0 -27
  73. data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,34 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/component"
12
13
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
14
 
15
- module Sidekiq
15
+ # module ScoutApm
16
+ # VERSION = "5.3.1"
17
+ # end
18
+ fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
+
20
+
21
+ scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
+
23
+
24
+ EOM
25
+
26
+ module Sidekiq # :nodoc:
16
27
  class CLI
17
- include Util
28
+ include Sidekiq::Component
18
29
  include Singleton unless $TESTING
19
30
 
20
31
  attr_accessor :launcher
21
32
  attr_accessor :environment
33
+ attr_accessor :config
34
+
35
+ def parse(args = ARGV.dup)
36
+ @config = Sidekiq
37
+ @config[:error_handlers].clear
38
+ @config[:error_handlers] << @config.method(:default_error_handler)
22
39
 
23
- def parse(args = ARGV)
24
40
  setup_options(args)
25
41
  initialize_logger
26
42
  validate!
@@ -36,7 +52,7 @@ module Sidekiq
36
52
  def run(boot_app: true)
37
53
  boot_application if boot_app
38
54
 
39
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
55
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
40
56
  print_banner
41
57
  end
42
58
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +83,7 @@ module Sidekiq
67
83
 
68
84
  # touch the connection pool so it is created before we
69
85
  # fire startup and start multithreading.
70
- info = Sidekiq.redis_info
86
+ info = @config.redis_info
71
87
  ver = info["redis_version"]
72
88
  raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
73
89
 
@@ -85,22 +101,22 @@ module Sidekiq
85
101
 
86
102
  # Since the user can pass us a connection pool explicitly in the initializer, we
87
103
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
88
- cursize = Sidekiq.redis_pool.size
89
- needed = Sidekiq.options[:concurrency] + 2
104
+ cursize = @config.redis_pool.size
105
+ needed = @config[:concurrency] + 2
90
106
  raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
91
107
 
92
108
  # cache process identity
93
- Sidekiq.options[:identity] = identity
109
+ @config[:identity] = identity
94
110
 
95
111
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
96
- Sidekiq.server_middleware
112
+ @config.server_middleware
97
113
 
98
114
  # Before this point, the process is initializing with just the main thread.
99
115
  # Starting here the process will now have multiple threads running.
100
116
  fire_event(:startup, reverse: false, reraise: true)
101
117
 
102
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
103
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
118
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
119
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
104
120
 
105
121
  launch(self_read)
106
122
  end
@@ -110,13 +126,13 @@ module Sidekiq
110
126
  logger.info "Starting processing, hit Ctrl-C to stop"
111
127
  end
112
128
 
113
- @launcher = Sidekiq::Launcher.new(options)
129
+ @launcher = Sidekiq::Launcher.new(@config)
114
130
 
115
131
  begin
116
132
  launcher.run
117
133
 
118
- while (readable_io = IO.select([self_read]))
119
- signal = readable_io.first[0].gets.strip
134
+ while self_read.wait_readable
135
+ signal = self_read.gets.strip
120
136
  handle_signal(signal)
121
137
  end
122
138
  rescue Interrupt
@@ -173,25 +189,25 @@ module Sidekiq
173
189
  # Heroku sends TERM and then waits 30 seconds for process to exit.
174
190
  "TERM" => ->(cli) { raise Interrupt },
175
191
  "TSTP" => ->(cli) {
176
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
192
+ cli.logger.info "Received TSTP, no longer accepting new work"
177
193
  cli.launcher.quiet
178
194
  },
179
195
  "TTIN" => ->(cli) {
180
196
  Thread.list.each do |thread|
181
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
197
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
182
198
  if thread.backtrace
183
- Sidekiq.logger.warn thread.backtrace.join("\n")
199
+ cli.logger.warn thread.backtrace.join("\n")
184
200
  else
185
- Sidekiq.logger.warn "<no backtrace available>"
201
+ cli.logger.warn "<no backtrace available>"
186
202
  end
187
203
  end
188
204
  }
189
205
  }
190
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
206
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
191
207
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
192
208
 
193
209
  def handle_signal(sig)
194
- Sidekiq.logger.debug "Got #{sig} signal"
210
+ logger.debug "Got #{sig} signal"
195
211
  SIGNAL_HANDLERS[sig].call(self)
196
212
  end
197
213
 
@@ -209,6 +225,7 @@ module Sidekiq
209
225
  # Both Sinatra 2.0+ and Sidekiq support this term.
210
226
  # RAILS_ENV and RACK_ENV are there for legacy support.
211
227
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
+ config[:environment] = @environment
212
229
  end
213
230
 
214
231
  def symbolize_keys_deep!(hash)
@@ -237,7 +254,7 @@ module Sidekiq
237
254
  config_dir = if File.directory?(opts[:require].to_s)
238
255
  File.join(opts[:require], "config")
239
256
  else
240
- File.join(options[:require], "config")
257
+ File.join(@config[:require], "config")
241
258
  end
242
259
 
243
260
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +271,23 @@ module Sidekiq
254
271
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
255
272
 
256
273
  # merge with defaults
257
- options.merge!(opts)
258
- end
259
-
260
- def options
261
- Sidekiq.options
274
+ @config.merge!(opts)
262
275
  end
263
276
 
264
277
  def boot_application
265
278
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
266
279
 
267
- if File.directory?(options[:require])
280
+ if File.directory?(@config[:require])
268
281
  require "rails"
269
282
  if ::Rails::VERSION::MAJOR < 5
270
283
  raise "Sidekiq no longer supports this version of Rails"
271
284
  else
272
285
  require "sidekiq/rails"
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
286
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
274
287
  end
275
- options[:tag] ||= default_tag
288
+ @config[:tag] ||= default_tag
276
289
  else
277
- require options[:require]
290
+ require @config[:require]
278
291
  end
279
292
  end
280
293
 
@@ -291,18 +304,18 @@ module Sidekiq
291
304
  end
292
305
 
293
306
  def validate!
294
- if !File.exist?(options[:require]) ||
295
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
307
+ if !File.exist?(@config[:require]) ||
308
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
296
309
  logger.info "=================================================================="
297
310
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
298
- logger.info " to load your worker classes with -r [DIR|FILE]."
311
+ logger.info " to load your job classes with -r [DIR|FILE]."
299
312
  logger.info "=================================================================="
300
313
  logger.info @parser
301
314
  die(1)
302
315
  end
303
316
 
304
317
  [:concurrency, :timeout].each do |opt|
305
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
318
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
306
319
  end
307
320
  end
308
321
 
@@ -336,7 +349,7 @@ module Sidekiq
336
349
  parse_queue opts, queue, weight
337
350
  end
338
351
 
339
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
352
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
340
353
  opts[:require] = arg
341
354
  end
342
355
 
@@ -376,13 +389,13 @@ module Sidekiq
376
389
  end
377
390
 
378
391
  def initialize_logger
379
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
392
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
380
393
  end
381
394
 
382
395
  def parse_config(path)
383
396
  erb = ERB.new(File.read(path))
384
397
  erb.filename = File.expand_path(path)
385
- opts = YAML.load(erb.result) || {}
398
+ opts = load_yaml(erb.result) || {}
386
399
 
387
400
  if opts.respond_to? :deep_symbolize_keys!
388
401
  opts.deep_symbolize_keys!
@@ -398,6 +411,14 @@ module Sidekiq
398
411
  opts
399
412
  end
400
413
 
414
+ def load_yaml(src)
415
+ if Psych::VERSION > "4.0"
416
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
417
+ else
418
+ YAML.load(src)
419
+ end
420
+ end
421
+
401
422
  def parse_queues(opts, queues_and_weights)
402
423
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
403
424
  end
@@ -417,3 +438,4 @@ module Sidekiq
417
438
  end
418
439
 
419
440
  require "sidekiq/systemd"
441
+ require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
@@ -15,7 +15,7 @@ module Sidekiq
15
15
  # client.middleware do |chain|
16
16
  # chain.use MyClientMiddleware
17
17
  # end
18
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
19
19
  #
20
20
  # All client instances default to the globally-defined
21
21
  # Sidekiq.client_middleware but you can change as necessary.
@@ -49,16 +49,16 @@ module Sidekiq
49
49
  # The main method used to push a job to Redis. Accepts a number of options:
50
50
  #
51
51
  # queue - the named queue to use, default 'default'
52
- # class - the worker class to call, required
52
+ # class - the job class to call, required
53
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
54
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
55
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
56
56
  # backtrace - whether to save any error backtrace, default false
57
57
  #
58
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
59
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
60
60
  #
61
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
62
62
  #
63
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
64
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -67,13 +67,15 @@ module Sidekiq
67
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
68
68
  #
69
69
  # Example:
70
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
71
71
  #
72
72
  def push(item)
73
73
  normed = normalize_item(item)
74
- payload = process_single(item["class"], normed)
75
-
74
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
76
77
  if payload
78
+ verify_json(payload)
77
79
  raw_push([payload])
78
80
  payload["jid"]
79
81
  end
@@ -101,12 +103,17 @@ module Sidekiq
101
103
  raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
102
104
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
103
105
 
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
108
+
104
109
  normed = normalize_item(items)
105
110
  payloads = args.map.with_index { |job_args, index|
106
- copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
107
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
108
-
109
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
110
117
  result || nil
111
118
  }.compact
112
119
 
@@ -119,8 +126,8 @@ module Sidekiq
119
126
  #
120
127
  # pool = ConnectionPool.new { Redis.new }
121
128
  # Sidekiq::Client.via(pool) do
122
- # SomeWorker.perform_async(1,2,3)
123
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
124
131
  # end
125
132
  #
126
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -145,10 +152,10 @@ module Sidekiq
145
152
  end
146
153
 
147
154
  # Resque compatibility helpers. Note all helpers
148
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
149
156
  #
150
157
  # Example usage:
151
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
152
159
  #
153
160
  # Messages are enqueued to the 'default' queue.
154
161
  #
@@ -157,19 +164,19 @@ module Sidekiq
157
164
  end
158
165
 
159
166
  # Example usage:
160
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
161
168
  #
162
169
  def enqueue_to(queue, klass, *args)
163
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
164
171
  end
165
172
 
166
173
  # Example usage:
167
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
168
175
  #
169
176
  def enqueue_to_in(queue, interval, klass, *args)
170
177
  int = interval.to_f
171
178
  now = Time.now.to_f
172
- ts = (int < 1_000_000_000 ? now + int : int)
179
+ ts = ((int < 1_000_000_000) ? now + int : int)
173
180
 
174
181
  item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
175
182
  item.delete("at") if ts <= now
@@ -178,7 +185,7 @@ module Sidekiq
178
185
  end
179
186
 
180
187
  # Example usage:
181
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
182
189
  #
183
190
  def enqueue_in(interval, klass, *args)
184
191
  klass.perform_in(interval, *args)
@@ -189,8 +196,23 @@ module Sidekiq
189
196
 
190
197
  def raw_push(payloads)
191
198
  @redis_pool.with do |conn|
192
- conn.pipelined do
193
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue RedisConnection.adapter::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
194
216
  end
195
217
  end
196
218
  true
@@ -198,7 +220,7 @@ module Sidekiq
198
220
 
199
221
  def atomic_push(conn, payloads)
200
222
  if payloads.first.key?("at")
201
- conn.zadd("schedule", payloads.map { |hash|
223
+ conn.zadd("schedule", payloads.flat_map { |hash|
202
224
  at = hash.delete("at").to_s
203
225
  [at, Sidekiq.dump_json(hash)]
204
226
  })
@@ -209,17 +231,9 @@ module Sidekiq
209
231
  entry["enqueued_at"] = now
210
232
  Sidekiq.dump_json(entry)
211
233
  }
212
- conn.sadd("queues", queue)
234
+ conn.sadd("queues", [queue])
213
235
  conn.lpush("queue:#{queue}", to_push)
214
236
  end
215
237
  end
216
-
217
- def process_single(worker_class, item)
218
- queue = item["queue"]
219
-
220
- middleware.invoke(worker_class, item, queue, @redis_pool) do
221
- item
222
- end
223
- end
224
238
  end
225
239
  end
@@ -0,0 +1,65 @@
1
+ module Sidekiq
2
+ ##
3
+ # Sidekiq::Component assumes a config instance is available at @config
4
+ module Component # :nodoc:
5
+ attr_reader :config
6
+
7
+ def watchdog(last_words)
8
+ yield
9
+ rescue Exception => ex
10
+ handle_exception(ex, {context: last_words})
11
+ raise ex
12
+ end
13
+
14
+ def safe_thread(name, &block)
15
+ Thread.new do
16
+ Thread.current.name = name
17
+ watchdog(name, &block)
18
+ end
19
+ end
20
+
21
+ def logger
22
+ config.logger
23
+ end
24
+
25
+ def redis(&block)
26
+ config.redis(&block)
27
+ end
28
+
29
+ def tid
30
+ Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
+ end
32
+
33
+ def hostname
34
+ ENV["DYNO"] || Socket.gethostname
35
+ end
36
+
37
+ def process_nonce
38
+ @@process_nonce ||= SecureRandom.hex(6)
39
+ end
40
+
41
+ def identity
42
+ @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
43
+ end
44
+
45
+ def handle_exception(ex, ctx = {})
46
+ config.handle_exception(ex, ctx)
47
+ end
48
+
49
+ def fire_event(event, options = {})
50
+ oneshot = options.fetch(:oneshot, true)
51
+ reverse = options[:reverse]
52
+ reraise = options[:reraise]
53
+
54
+ arr = config[:lifecycle_events][event]
55
+ arr.reverse! if reverse
56
+ arr.each do |block|
57
+ block.call
58
+ rescue => ex
59
+ handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
60
+ raise ex if reraise
61
+ end
62
+ arr.clear if oneshot # once we've fired an event, we never fire it again
63
+ end
64
+ end
65
+ end
data/lib/sidekiq/delay.rb CHANGED
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Sidekiq
3
+ module Sidekiq # :nodoc:
4
4
  module Extensions
5
5
  def self.enable_delay!
6
- Sidekiq.logger.error "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0. #{caller(1..1).first}"
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
7
 
8
8
  if defined?(::ActiveSupport)
9
9
  require "sidekiq/extensions/active_record"
@@ -16,8 +16,8 @@ module Sidekiq
16
16
  include Sidekiq::Worker
17
17
 
18
18
  def perform(yml)
19
- (target, method_name, args, kwargs) = YAML.load(yml)
20
- msg = kwargs.empty? ? target.public_send(method_name, *args) : target.public_send(method_name, *args, **kwargs)
19
+ (target, method_name, args) = YAML.load(yml)
20
+ msg = target.public_send(method_name, *args)
21
21
  # The email method can return nil, which causes ActionMailer to return
22
22
  # an undeliverable empty message.
23
23
  if msg
@@ -18,8 +18,8 @@ module Sidekiq
18
18
  include Sidekiq::Worker
19
19
 
20
20
  def perform(yml)
21
- (target, method_name, args, kwargs) = YAML.load(yml)
22
- kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
21
+ (target, method_name, args) = YAML.load(yml)
22
+ target.__send__(method_name, *args)
23
23
  end
24
24
  end
25
25
 
@@ -16,8 +16,8 @@ module Sidekiq
16
16
  include Sidekiq::Worker
17
17
 
18
18
  def perform(yml)
19
- (target, method_name, args, kwargs) = YAML.load(yml)
20
- kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
19
+ (target, method_name, args) = YAML.load(yml)
20
+ target.__send__(method_name, *args)
21
21
  end
22
22
  end
23
23
 
@@ -10,16 +10,16 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
- def method_missing(name, *args, **kwargs)
16
+ def method_missing(name, *args)
17
17
  # Sidekiq has a limitation in that its message must be JSON.
18
18
  # JSON can't round trip real Ruby objects so we use YAML to
19
19
  # serialize the objects to a String. The YAML will be converted
20
20
  # to JSON and then deserialized on the other side back into a
21
21
  # Ruby object.
22
- obj = [@target, name, args, kwargs]
22
+ obj = [@target, name, args]
23
23
  marshalled = ::YAML.dump(obj)
24
24
  if marshalled.size > SIZE_LIMIT
25
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
data/lib/sidekiq/fetch.rb CHANGED
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
+ require "sidekiq/component"
4
5
 
5
- module Sidekiq
6
+ module Sidekiq # :nodoc:
6
7
  class BasicFetch
8
+ include Sidekiq::Component
7
9
  # We want the fetch operation to timeout every few seconds so the thread
8
10
  # can check if the process is shutting down.
9
11
  TIMEOUT = 2
10
12
 
11
- UnitOfWork = Struct.new(:queue, :job) {
13
+ UnitOfWork = Struct.new(:queue, :job, :config) {
12
14
  def acknowledge
13
15
  # nothing to do
14
16
  end
@@ -18,20 +20,20 @@ module Sidekiq
18
20
  end
19
21
 
20
22
  def requeue
21
- Sidekiq.redis do |conn|
23
+ config.redis do |conn|
22
24
  conn.rpush(queue, job)
23
25
  end
24
26
  end
25
27
  }
26
28
 
27
- def initialize(options)
28
- raise ArgumentError, "missing queue list" unless options[:queues]
29
- @options = options
30
- @strictly_ordered_queues = !!@options[:strict]
31
- @queues = @options[:queues].map { |q| "queue:#{q}" }
29
+ def initialize(config)
30
+ raise ArgumentError, "missing queue list" unless config[:queues]
31
+ @config = config
32
+ @strictly_ordered_queues = !!@config[:strict]
33
+ @queues = @config[:queues].map { |q| "queue:#{q}" }
32
34
  if @strictly_ordered_queues
33
35
  @queues.uniq!
34
- @queues << TIMEOUT
36
+ @queues << {timeout: TIMEOUT}
35
37
  end
36
38
  end
37
39
 
@@ -44,30 +46,30 @@ module Sidekiq
44
46
  return nil
45
47
  end
46
48
 
47
- work = Sidekiq.redis { |conn| conn.brpop(*qs) }
48
- UnitOfWork.new(*work) if work
49
+ queue, job = redis { |conn| conn.brpop(*qs) }
50
+ UnitOfWork.new(queue, job, config) if queue
49
51
  end
50
52
 
51
53
  def bulk_requeue(inprogress, options)
52
54
  return if inprogress.empty?
53
55
 
54
- Sidekiq.logger.debug { "Re-queueing terminated jobs" }
56
+ logger.debug { "Re-queueing terminated jobs" }
55
57
  jobs_to_requeue = {}
56
58
  inprogress.each do |unit_of_work|
57
59
  jobs_to_requeue[unit_of_work.queue] ||= []
58
60
  jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
59
61
  end
60
62
 
61
- Sidekiq.redis do |conn|
62
- conn.pipelined do
63
+ redis do |conn|
64
+ conn.pipelined do |pipeline|
63
65
  jobs_to_requeue.each do |queue, jobs|
64
- conn.rpush(queue, jobs)
66
+ pipeline.rpush(queue, jobs)
65
67
  end
66
68
  end
67
69
  end
68
- Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
70
+ logger.info("Pushed #{inprogress.size} jobs back to Redis")
69
71
  rescue => ex
70
- Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
72
+ logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
71
73
  end
72
74
 
73
75
  # Creating the Redis#brpop command takes into account any
@@ -81,7 +83,7 @@ module Sidekiq
81
83
  else
82
84
  permute = @queues.shuffle
83
85
  permute.uniq!
84
- permute << TIMEOUT
86
+ permute << {timeout: TIMEOUT}
85
87
  permute
86
88
  end
87
89
  end