sidekiq 6.4.0 → 6.5.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +50 -1
  3. data/README.md +6 -1
  4. data/bin/sidekiq +3 -3
  5. data/bin/sidekiqload +70 -66
  6. data/bin/sidekiqmon +1 -1
  7. data/lib/sidekiq/api.rb +78 -65
  8. data/lib/sidekiq/cli.rb +46 -37
  9. data/lib/sidekiq/client.rb +42 -28
  10. data/lib/sidekiq/component.rb +64 -0
  11. data/lib/sidekiq/delay.rb +1 -1
  12. data/lib/sidekiq/extensions/action_mailer.rb +2 -2
  13. data/lib/sidekiq/extensions/active_record.rb +2 -2
  14. data/lib/sidekiq/extensions/class_methods.rb +2 -2
  15. data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
  16. data/lib/sidekiq/fetch.rb +17 -15
  17. data/lib/sidekiq/job_logger.rb +15 -27
  18. data/lib/sidekiq/job_retry.rb +27 -26
  19. data/lib/sidekiq/job_util.rb +15 -9
  20. data/lib/sidekiq/launcher.rb +54 -52
  21. data/lib/sidekiq/logger.rb +8 -18
  22. data/lib/sidekiq/manager.rb +28 -25
  23. data/lib/sidekiq/middleware/chain.rb +22 -13
  24. data/lib/sidekiq/middleware/current_attributes.rb +4 -0
  25. data/lib/sidekiq/middleware/i18n.rb +6 -4
  26. data/lib/sidekiq/middleware/modules.rb +19 -0
  27. data/lib/sidekiq/monitor.rb +1 -1
  28. data/lib/sidekiq/paginator.rb +8 -8
  29. data/lib/sidekiq/processor.rb +38 -38
  30. data/lib/sidekiq/rails.rb +15 -8
  31. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  32. data/lib/sidekiq/redis_connection.rb +81 -48
  33. data/lib/sidekiq/ring_buffer.rb +29 -0
  34. data/lib/sidekiq/scheduled.rb +11 -10
  35. data/lib/sidekiq/testing/inline.rb +4 -4
  36. data/lib/sidekiq/testing.rb +37 -36
  37. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  38. data/lib/sidekiq/version.rb +1 -1
  39. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  40. data/lib/sidekiq/web/helpers.rb +5 -5
  41. data/lib/sidekiq/web.rb +3 -3
  42. data/lib/sidekiq/worker.rb +20 -17
  43. data/lib/sidekiq.rb +98 -30
  44. data/web/assets/javascripts/application.js +58 -26
  45. data/web/assets/stylesheets/application.css +1 -0
  46. data/web/locales/pt-br.yml +27 -9
  47. data/web/views/_summary.erb +1 -1
  48. data/web/views/busy.erb +3 -3
  49. metadata +8 -5
  50. data/lib/sidekiq/exception_handler.rb +0 -27
  51. data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,23 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/component"
12
13
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
14
 
15
15
  module Sidekiq
16
16
  class CLI
17
- include Util
17
+ include Sidekiq::Component
18
18
  include Singleton unless $TESTING
19
19
 
20
20
  attr_accessor :launcher
21
21
  attr_accessor :environment
22
+ attr_accessor :config
23
+
24
+ def parse(args = ARGV.dup)
25
+ @config = Sidekiq
26
+ @config[:error_handlers].clear
27
+ @config[:error_handlers] << @config.method(:default_error_handler)
22
28
 
23
- def parse(args = ARGV)
24
29
  setup_options(args)
25
30
  initialize_logger
26
31
  validate!
@@ -36,7 +41,7 @@ module Sidekiq
36
41
  def run(boot_app: true)
37
42
  boot_application if boot_app
38
43
 
39
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
44
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
40
45
  print_banner
41
46
  end
42
47
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +72,7 @@ module Sidekiq
67
72
 
68
73
  # touch the connection pool so it is created before we
69
74
  # fire startup and start multithreading.
70
- info = Sidekiq.redis_info
75
+ info = @config.redis_info
71
76
  ver = info["redis_version"]
72
77
  raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
73
78
 
@@ -85,22 +90,22 @@ module Sidekiq
85
90
 
86
91
  # Since the user can pass us a connection pool explicitly in the initializer, we
87
92
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
88
- cursize = Sidekiq.redis_pool.size
89
- needed = Sidekiq.options[:concurrency] + 2
93
+ cursize = @config.redis_pool.size
94
+ needed = @config[:concurrency] + 2
90
95
  raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
91
96
 
92
97
  # cache process identity
93
- Sidekiq.options[:identity] = identity
98
+ @config[:identity] = identity
94
99
 
95
100
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
96
- Sidekiq.server_middleware
101
+ @config.server_middleware
97
102
 
98
103
  # Before this point, the process is initializing with just the main thread.
99
104
  # Starting here the process will now have multiple threads running.
100
105
  fire_event(:startup, reverse: false, reraise: true)
101
106
 
102
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
103
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
107
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
108
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
104
109
 
105
110
  launch(self_read)
106
111
  end
@@ -110,13 +115,13 @@ module Sidekiq
110
115
  logger.info "Starting processing, hit Ctrl-C to stop"
111
116
  end
112
117
 
113
- @launcher = Sidekiq::Launcher.new(options)
118
+ @launcher = Sidekiq::Launcher.new(@config)
114
119
 
115
120
  begin
116
121
  launcher.run
117
122
 
118
- while (readable_io = IO.select([self_read]))
119
- signal = readable_io.first[0].gets.strip
123
+ while self_read.wait_readable
124
+ signal = self_read.gets.strip
120
125
  handle_signal(signal)
121
126
  end
122
127
  rescue Interrupt
@@ -173,25 +178,25 @@ module Sidekiq
173
178
  # Heroku sends TERM and then waits 30 seconds for process to exit.
174
179
  "TERM" => ->(cli) { raise Interrupt },
175
180
  "TSTP" => ->(cli) {
176
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
181
+ cli.logger.info "Received TSTP, no longer accepting new work"
177
182
  cli.launcher.quiet
178
183
  },
179
184
  "TTIN" => ->(cli) {
180
185
  Thread.list.each do |thread|
181
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
186
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
182
187
  if thread.backtrace
183
- Sidekiq.logger.warn thread.backtrace.join("\n")
188
+ cli.logger.warn thread.backtrace.join("\n")
184
189
  else
185
- Sidekiq.logger.warn "<no backtrace available>"
190
+ cli.logger.warn "<no backtrace available>"
186
191
  end
187
192
  end
188
193
  }
189
194
  }
190
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
195
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
191
196
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
192
197
 
193
198
  def handle_signal(sig)
194
- Sidekiq.logger.debug "Got #{sig} signal"
199
+ logger.debug "Got #{sig} signal"
195
200
  SIGNAL_HANDLERS[sig].call(self)
196
201
  end
197
202
 
@@ -237,7 +242,7 @@ module Sidekiq
237
242
  config_dir = if File.directory?(opts[:require].to_s)
238
243
  File.join(opts[:require], "config")
239
244
  else
240
- File.join(options[:require], "config")
245
+ File.join(@config[:require], "config")
241
246
  end
242
247
 
243
248
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +259,23 @@ module Sidekiq
254
259
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
255
260
 
256
261
  # merge with defaults
257
- options.merge!(opts)
258
- end
259
-
260
- def options
261
- Sidekiq.options
262
+ @config.merge!(opts)
262
263
  end
263
264
 
264
265
  def boot_application
265
266
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
266
267
 
267
- if File.directory?(options[:require])
268
+ if File.directory?(@config[:require])
268
269
  require "rails"
269
270
  if ::Rails::VERSION::MAJOR < 5
270
271
  raise "Sidekiq no longer supports this version of Rails"
271
272
  else
272
273
  require "sidekiq/rails"
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
274
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
274
275
  end
275
- options[:tag] ||= default_tag
276
+ @config[:tag] ||= default_tag
276
277
  else
277
- require options[:require]
278
+ require @config[:require]
278
279
  end
279
280
  end
280
281
 
@@ -291,18 +292,18 @@ module Sidekiq
291
292
  end
292
293
 
293
294
  def validate!
294
- if !File.exist?(options[:require]) ||
295
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
295
+ if !File.exist?(@config[:require]) ||
296
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
296
297
  logger.info "=================================================================="
297
298
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
298
- logger.info " to load your worker classes with -r [DIR|FILE]."
299
+ logger.info " to load your job classes with -r [DIR|FILE]."
299
300
  logger.info "=================================================================="
300
301
  logger.info @parser
301
302
  die(1)
302
303
  end
303
304
 
304
305
  [:concurrency, :timeout].each do |opt|
305
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
306
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
306
307
  end
307
308
  end
308
309
 
@@ -336,7 +337,7 @@ module Sidekiq
336
337
  parse_queue opts, queue, weight
337
338
  end
338
339
 
339
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
340
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
340
341
  opts[:require] = arg
341
342
  end
342
343
 
@@ -376,13 +377,13 @@ module Sidekiq
376
377
  end
377
378
 
378
379
  def initialize_logger
379
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
380
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
380
381
  end
381
382
 
382
383
  def parse_config(path)
383
384
  erb = ERB.new(File.read(path))
384
385
  erb.filename = File.expand_path(path)
385
- opts = YAML.load(erb.result) || {}
386
+ opts = load_yaml(erb.result) || {}
386
387
 
387
388
  if opts.respond_to? :deep_symbolize_keys!
388
389
  opts.deep_symbolize_keys!
@@ -398,6 +399,14 @@ module Sidekiq
398
399
  opts
399
400
  end
400
401
 
402
+ def load_yaml(src)
403
+ if Psych::VERSION > "4.0"
404
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
405
+ else
406
+ YAML.load(src)
407
+ end
408
+ end
409
+
401
410
  def parse_queues(opts, queues_and_weights)
402
411
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
403
412
  end
@@ -15,7 +15,7 @@ module Sidekiq
15
15
  # client.middleware do |chain|
16
16
  # chain.use MyClientMiddleware
17
17
  # end
18
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
19
19
  #
20
20
  # All client instances default to the globally-defined
21
21
  # Sidekiq.client_middleware but you can change as necessary.
@@ -49,16 +49,16 @@ module Sidekiq
49
49
  # The main method used to push a job to Redis. Accepts a number of options:
50
50
  #
51
51
  # queue - the named queue to use, default 'default'
52
- # class - the worker class to call, required
52
+ # class - the job class to call, required
53
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
54
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
55
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
56
56
  # backtrace - whether to save any error backtrace, default false
57
57
  #
58
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
59
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
60
60
  #
61
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
62
62
  #
63
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
64
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -67,13 +67,15 @@ module Sidekiq
67
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
68
68
  #
69
69
  # Example:
70
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
71
71
  #
72
72
  def push(item)
73
73
  normed = normalize_item(item)
74
- payload = process_single(item["class"], normed)
75
-
74
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
76
77
  if payload
78
+ verify_json(payload)
77
79
  raw_push([payload])
78
80
  payload["jid"]
79
81
  end
@@ -101,12 +103,17 @@ module Sidekiq
101
103
  raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
102
104
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
103
105
 
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
108
+
104
109
  normed = normalize_item(items)
105
110
  payloads = args.map.with_index { |job_args, index|
106
- copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
107
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
108
-
109
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
110
117
  result || nil
111
118
  }.compact
112
119
 
@@ -119,8 +126,8 @@ module Sidekiq
119
126
  #
120
127
  # pool = ConnectionPool.new { Redis.new }
121
128
  # Sidekiq::Client.via(pool) do
122
- # SomeWorker.perform_async(1,2,3)
123
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
124
131
  # end
125
132
  #
126
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -145,10 +152,10 @@ module Sidekiq
145
152
  end
146
153
 
147
154
  # Resque compatibility helpers. Note all helpers
148
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
149
156
  #
150
157
  # Example usage:
151
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
152
159
  #
153
160
  # Messages are enqueued to the 'default' queue.
154
161
  #
@@ -157,14 +164,14 @@ module Sidekiq
157
164
  end
158
165
 
159
166
  # Example usage:
160
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
161
168
  #
162
169
  def enqueue_to(queue, klass, *args)
163
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
164
171
  end
165
172
 
166
173
  # Example usage:
167
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
168
175
  #
169
176
  def enqueue_to_in(queue, interval, klass, *args)
170
177
  int = interval.to_f
@@ -178,7 +185,7 @@ module Sidekiq
178
185
  end
179
186
 
180
187
  # Example usage:
181
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
182
189
  #
183
190
  def enqueue_in(interval, klass, *args)
184
191
  klass.perform_in(interval, *args)
@@ -189,8 +196,23 @@ module Sidekiq
189
196
 
190
197
  def raw_push(payloads)
191
198
  @redis_pool.with do |conn|
192
- conn.pipelined do
193
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue RedisConnection.adapter::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
194
216
  end
195
217
  end
196
218
  true
@@ -198,7 +220,7 @@ module Sidekiq
198
220
 
199
221
  def atomic_push(conn, payloads)
200
222
  if payloads.first.key?("at")
201
- conn.zadd("schedule", payloads.map { |hash|
223
+ conn.zadd("schedule", *payloads.map { |hash|
202
224
  at = hash.delete("at").to_s
203
225
  [at, Sidekiq.dump_json(hash)]
204
226
  })
@@ -213,13 +235,5 @@ module Sidekiq
213
235
  conn.lpush("queue:#{queue}", to_push)
214
236
  end
215
237
  end
216
-
217
- def process_single(worker_class, item)
218
- queue = item["queue"]
219
-
220
- middleware.invoke(worker_class, item, queue, @redis_pool) do
221
- item
222
- end
223
- end
224
238
  end
225
239
  end
@@ -0,0 +1,64 @@
1
+ module Sidekiq
2
+ ##
3
+ # Sidekiq::Component assumes a config instance is available at @config
4
+ module Component
5
+ attr_reader :config
6
+
7
+ def watchdog(last_words)
8
+ yield
9
+ rescue Exception => ex
10
+ handle_exception(ex, {context: last_words})
11
+ raise ex
12
+ end
13
+
14
+ def safe_thread(name, &block)
15
+ Thread.new do
16
+ Thread.current.name = name
17
+ watchdog(name, &block)
18
+ end
19
+ end
20
+
21
+ def logger
22
+ config.logger
23
+ end
24
+
25
+ def redis(&block)
26
+ config.redis(&block)
27
+ end
28
+
29
+ def tid
30
+ Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
+ end
32
+
33
+ def hostname
34
+ ENV["DYNO"] || Socket.gethostname
35
+ end
36
+
37
+ def process_nonce
38
+ @@process_nonce ||= SecureRandom.hex(6)
39
+ end
40
+
41
+ def identity
42
+ @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
43
+ end
44
+
45
+ def handle_exception(ex, ctx = {})
46
+ config.handle_exception(ex, ctx)
47
+ end
48
+
49
+ def fire_event(event, options = {})
50
+ reverse = options[:reverse]
51
+ reraise = options[:reraise]
52
+
53
+ arr = config[:lifecycle_events][event]
54
+ arr.reverse! if reverse
55
+ arr.each do |block|
56
+ block.call
57
+ rescue => ex
58
+ handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
59
+ raise ex if reraise
60
+ end
61
+ arr.clear # once we've fired an event, we never fire it again
62
+ end
63
+ end
64
+ end
data/lib/sidekiq/delay.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  module Sidekiq
4
4
  module Extensions
5
5
  def self.enable_delay!
6
- Sidekiq.logger.error "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0. #{caller(1..1).first}"
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
7
 
8
8
  if defined?(::ActiveSupport)
9
9
  require "sidekiq/extensions/active_record"
@@ -16,8 +16,8 @@ module Sidekiq
16
16
  include Sidekiq::Worker
17
17
 
18
18
  def perform(yml)
19
- (target, method_name, args, kwargs) = YAML.load(yml)
20
- msg = kwargs.empty? ? target.public_send(method_name, *args) : target.public_send(method_name, *args, **kwargs)
19
+ (target, method_name, args) = YAML.load(yml)
20
+ msg = target.public_send(method_name, *args)
21
21
  # The email method can return nil, which causes ActionMailer to return
22
22
  # an undeliverable empty message.
23
23
  if msg
@@ -18,8 +18,8 @@ module Sidekiq
18
18
  include Sidekiq::Worker
19
19
 
20
20
  def perform(yml)
21
- (target, method_name, args, kwargs) = YAML.load(yml)
22
- kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
21
+ (target, method_name, args) = YAML.load(yml)
22
+ target.__send__(method_name, *args)
23
23
  end
24
24
  end
25
25
 
@@ -16,8 +16,8 @@ module Sidekiq
16
16
  include Sidekiq::Worker
17
17
 
18
18
  def perform(yml)
19
- (target, method_name, args, kwargs) = YAML.load(yml)
20
- kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
19
+ (target, method_name, args) = YAML.load(yml)
20
+ target.__send__(method_name, *args)
21
21
  end
22
22
  end
23
23
 
@@ -10,16 +10,16 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
- def method_missing(name, *args, **kwargs)
16
+ def method_missing(name, *args)
17
17
  # Sidekiq has a limitation in that its message must be JSON.
18
18
  # JSON can't round trip real Ruby objects so we use YAML to
19
19
  # serialize the objects to a String. The YAML will be converted
20
20
  # to JSON and then deserialized on the other side back into a
21
21
  # Ruby object.
22
- obj = [@target, name, args, kwargs]
22
+ obj = [@target, name, args]
23
23
  marshalled = ::YAML.dump(obj)
24
24
  if marshalled.size > SIZE_LIMIT
25
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
data/lib/sidekiq/fetch.rb CHANGED
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
+ require "sidekiq/component"
4
5
 
5
6
  module Sidekiq
6
7
  class BasicFetch
8
+ include Sidekiq::Component
7
9
  # We want the fetch operation to timeout every few seconds so the thread
8
10
  # can check if the process is shutting down.
9
11
  TIMEOUT = 2
10
12
 
11
- UnitOfWork = Struct.new(:queue, :job) {
13
+ UnitOfWork = Struct.new(:queue, :job, :config) {
12
14
  def acknowledge
13
15
  # nothing to do
14
16
  end
@@ -18,17 +20,17 @@ module Sidekiq
18
20
  end
19
21
 
20
22
  def requeue
21
- Sidekiq.redis do |conn|
23
+ config.redis do |conn|
22
24
  conn.rpush(queue, job)
23
25
  end
24
26
  end
25
27
  }
26
28
 
27
- def initialize(options)
28
- raise ArgumentError, "missing queue list" unless options[:queues]
29
- @options = options
30
- @strictly_ordered_queues = !!@options[:strict]
31
- @queues = @options[:queues].map { |q| "queue:#{q}" }
29
+ def initialize(config)
30
+ raise ArgumentError, "missing queue list" unless config[:queues]
31
+ @config = config
32
+ @strictly_ordered_queues = !!@config[:strict]
33
+ @queues = @config[:queues].map { |q| "queue:#{q}" }
32
34
  if @strictly_ordered_queues
33
35
  @queues.uniq!
34
36
  @queues << TIMEOUT
@@ -44,30 +46,30 @@ module Sidekiq
44
46
  return nil
45
47
  end
46
48
 
47
- work = Sidekiq.redis { |conn| conn.brpop(*qs) }
48
- UnitOfWork.new(*work) if work
49
+ queue, job = redis { |conn| conn.brpop(*qs) }
50
+ UnitOfWork.new(queue, job, config) if queue
49
51
  end
50
52
 
51
53
  def bulk_requeue(inprogress, options)
52
54
  return if inprogress.empty?
53
55
 
54
- Sidekiq.logger.debug { "Re-queueing terminated jobs" }
56
+ logger.debug { "Re-queueing terminated jobs" }
55
57
  jobs_to_requeue = {}
56
58
  inprogress.each do |unit_of_work|
57
59
  jobs_to_requeue[unit_of_work.queue] ||= []
58
60
  jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
59
61
  end
60
62
 
61
- Sidekiq.redis do |conn|
62
- conn.pipelined do
63
+ redis do |conn|
64
+ conn.pipelined do |pipeline|
63
65
  jobs_to_requeue.each do |queue, jobs|
64
- conn.rpush(queue, jobs)
66
+ pipeline.rpush(queue, jobs)
65
67
  end
66
68
  end
67
69
  end
68
- Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
70
+ logger.info("Pushed #{inprogress.size} jobs back to Redis")
69
71
  rescue => ex
70
- Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
72
+ logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
71
73
  end
72
74
 
73
75
  # Creating the Redis#brpop command takes into account any
@@ -12,46 +12,34 @@ module Sidekiq
12
12
 
13
13
  yield
14
14
 
15
- with_elapsed_time_context(start) do
16
- @logger.info("done")
17
- end
15
+ Sidekiq::Context.add(:elapsed, elapsed(start))
16
+ @logger.info("done")
18
17
  rescue Exception
19
- with_elapsed_time_context(start) do
20
- @logger.info("fail")
21
- end
18
+ Sidekiq::Context.add(:elapsed, elapsed(start))
19
+ @logger.info("fail")
22
20
 
23
21
  raise
24
22
  end
25
23
 
26
24
  def prepare(job_hash, &block)
27
- level = job_hash["log_level"]
28
- if level
29
- @logger.log_at(level) do
30
- Sidekiq::Context.with(job_hash_context(job_hash), &block)
31
- end
32
- else
33
- Sidekiq::Context.with(job_hash_context(job_hash), &block)
34
- end
35
- end
36
-
37
- def job_hash_context(job_hash)
38
25
  # If we're using a wrapper class, like ActiveJob, use the "wrapped"
39
26
  # attribute to expose the underlying thing.
40
27
  h = {
41
28
  class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
42
29
  jid: job_hash["jid"]
43
30
  }
44
- h[:bid] = job_hash["bid"] if job_hash["bid"]
45
- h[:tags] = job_hash["tags"] if job_hash["tags"]
46
- h
47
- end
48
-
49
- def with_elapsed_time_context(start, &block)
50
- Sidekiq::Context.with(elapsed_time_context(start), &block)
51
- end
31
+ h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
32
+ h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
52
33
 
53
- def elapsed_time_context(start)
54
- {elapsed: elapsed(start).to_s}
34
+ Thread.current[:sidekiq_context] = h
35
+ level = job_hash["log_level"]
36
+ if level
37
+ @logger.log_at(level, &block)
38
+ else
39
+ yield
40
+ end
41
+ ensure
42
+ Thread.current[:sidekiq_context] = nil
55
43
  end
56
44
 
57
45
  private