sidekiq 6.0.7 → 6.5.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +209 -2
  3. data/LICENSE +3 -3
  4. data/README.md +11 -10
  5. data/bin/sidekiq +8 -3
  6. data/bin/sidekiqload +70 -66
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +180 -123
  13. data/lib/sidekiq/cli.rb +80 -45
  14. data/lib/sidekiq/client.rb +52 -71
  15. data/lib/sidekiq/{util.rb → component.rb} +11 -14
  16. data/lib/sidekiq/delay.rb +2 -0
  17. data/lib/sidekiq/extensions/action_mailer.rb +3 -2
  18. data/lib/sidekiq/extensions/active_record.rb +4 -3
  19. data/lib/sidekiq/extensions/class_methods.rb +5 -4
  20. data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
  21. data/lib/sidekiq/fetch.rb +41 -30
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +16 -28
  24. data/lib/sidekiq/job_retry.rb +36 -36
  25. data/lib/sidekiq/job_util.rb +71 -0
  26. data/lib/sidekiq/launcher.rb +123 -63
  27. data/lib/sidekiq/logger.rb +11 -20
  28. data/lib/sidekiq/manager.rb +35 -34
  29. data/lib/sidekiq/middleware/chain.rb +28 -17
  30. data/lib/sidekiq/middleware/current_attributes.rb +61 -0
  31. data/lib/sidekiq/middleware/i18n.rb +6 -4
  32. data/lib/sidekiq/middleware/modules.rb +19 -0
  33. data/lib/sidekiq/monitor.rb +1 -1
  34. data/lib/sidekiq/paginator.rb +8 -8
  35. data/lib/sidekiq/processor.rb +41 -41
  36. data/lib/sidekiq/rails.rb +38 -22
  37. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  38. data/lib/sidekiq/redis_connection.rb +87 -53
  39. data/lib/sidekiq/ring_buffer.rb +29 -0
  40. data/lib/sidekiq/scheduled.rb +60 -24
  41. data/lib/sidekiq/sd_notify.rb +1 -1
  42. data/lib/sidekiq/testing/inline.rb +4 -4
  43. data/lib/sidekiq/testing.rb +39 -40
  44. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  45. data/lib/sidekiq/version.rb +1 -1
  46. data/lib/sidekiq/web/action.rb +2 -2
  47. data/lib/sidekiq/web/application.rb +21 -12
  48. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  49. data/lib/sidekiq/web/helpers.rb +40 -34
  50. data/lib/sidekiq/web/router.rb +5 -2
  51. data/lib/sidekiq/web.rb +36 -72
  52. data/lib/sidekiq/worker.rb +136 -16
  53. data/lib/sidekiq.rb +107 -30
  54. data/sidekiq.gemspec +11 -4
  55. data/web/assets/images/apple-touch-icon.png +0 -0
  56. data/web/assets/javascripts/application.js +113 -65
  57. data/web/assets/javascripts/dashboard.js +51 -51
  58. data/web/assets/stylesheets/application-dark.css +64 -43
  59. data/web/assets/stylesheets/application-rtl.css +0 -4
  60. data/web/assets/stylesheets/application.css +42 -239
  61. data/web/locales/ar.yml +8 -2
  62. data/web/locales/en.yml +4 -1
  63. data/web/locales/es.yml +18 -2
  64. data/web/locales/fr.yml +8 -1
  65. data/web/locales/ja.yml +3 -0
  66. data/web/locales/lt.yml +1 -1
  67. data/web/locales/pl.yml +4 -4
  68. data/web/locales/pt-br.yml +27 -9
  69. data/web/locales/ru.yml +4 -0
  70. data/web/views/_footer.erb +1 -1
  71. data/web/views/_job_info.erb +1 -1
  72. data/web/views/_poll_link.erb +2 -5
  73. data/web/views/_summary.erb +7 -7
  74. data/web/views/busy.erb +51 -20
  75. data/web/views/dashboard.erb +22 -14
  76. data/web/views/dead.erb +1 -1
  77. data/web/views/layout.erb +2 -1
  78. data/web/views/morgue.erb +6 -6
  79. data/web/views/queue.erb +11 -11
  80. data/web/views/queues.erb +4 -4
  81. data/web/views/retries.erb +7 -7
  82. data/web/views/retry.erb +1 -1
  83. data/web/views/scheduled.erb +1 -1
  84. metadata +29 -51
  85. data/.circleci/config.yml +0 -60
  86. data/.github/contributing.md +0 -32
  87. data/.github/issue_template.md +0 -11
  88. data/.gitignore +0 -13
  89. data/.standard.yml +0 -20
  90. data/3.0-Upgrade.md +0 -70
  91. data/4.0-Upgrade.md +0 -53
  92. data/5.0-Upgrade.md +0 -56
  93. data/6.0-Upgrade.md +0 -72
  94. data/COMM-LICENSE +0 -97
  95. data/Ent-2.0-Upgrade.md +0 -37
  96. data/Ent-Changes.md +0 -256
  97. data/Gemfile +0 -24
  98. data/Gemfile.lock +0 -208
  99. data/Pro-2.0-Upgrade.md +0 -138
  100. data/Pro-3.0-Upgrade.md +0 -44
  101. data/Pro-4.0-Upgrade.md +0 -35
  102. data/Pro-5.0-Upgrade.md +0 -25
  103. data/Pro-Changes.md +0 -782
  104. data/Rakefile +0 -10
  105. data/code_of_conduct.md +0 -50
  106. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  107. data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,23 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/component"
12
13
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
14
 
15
15
  module Sidekiq
16
16
  class CLI
17
- include Util
17
+ include Sidekiq::Component
18
18
  include Singleton unless $TESTING
19
19
 
20
20
  attr_accessor :launcher
21
21
  attr_accessor :environment
22
+ attr_accessor :config
23
+
24
+ def parse(args = ARGV.dup)
25
+ @config = Sidekiq
26
+ @config[:error_handlers].clear
27
+ @config[:error_handlers] << @config.method(:default_error_handler)
22
28
 
23
- def parse(args = ARGV)
24
29
  setup_options(args)
25
30
  initialize_logger
26
31
  validate!
@@ -33,9 +38,10 @@ module Sidekiq
33
38
  # Code within this method is not tested because it alters
34
39
  # global process state irreversibly. PRs which improve the
35
40
  # test coverage of Sidekiq::CLI are welcomed.
36
- def run
37
- boot_system
38
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
41
+ def run(boot_app: true)
42
+ boot_application if boot_app
43
+
44
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
39
45
  print_banner
40
46
  end
41
47
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -43,9 +49,17 @@ module Sidekiq
43
49
  self_read, self_write = IO.pipe
44
50
  sigs = %w[INT TERM TTIN TSTP]
45
51
  # USR1 and USR2 don't work on the JVM
46
- sigs << "USR2" unless jruby?
52
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
47
53
  sigs.each do |sig|
48
- trap sig do
54
+ old_handler = Signal.trap(sig) do
55
+ if old_handler.respond_to?(:call)
56
+ begin
57
+ old_handler.call
58
+ rescue Exception => exc
59
+ # signal handlers can't use Logger so puts only
60
+ puts ["Error in #{sig} handler", exc].inspect
61
+ end
62
+ end
49
63
  self_write.puts(sig)
50
64
  end
51
65
  rescue ArgumentError
@@ -58,27 +72,40 @@ module Sidekiq
58
72
 
59
73
  # touch the connection pool so it is created before we
60
74
  # fire startup and start multithreading.
61
- ver = Sidekiq.redis_info["redis_version"]
75
+ info = @config.redis_info
76
+ ver = info["redis_version"]
62
77
  raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
63
78
 
79
+ maxmemory_policy = info["maxmemory_policy"]
80
+ if maxmemory_policy != "noeviction"
81
+ logger.warn <<~EOM
82
+
83
+
84
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
85
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
86
+ See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
87
+
88
+ EOM
89
+ end
90
+
64
91
  # Since the user can pass us a connection pool explicitly in the initializer, we
65
92
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
66
- cursize = Sidekiq.redis_pool.size
67
- needed = Sidekiq.options[:concurrency] + 2
93
+ cursize = @config.redis_pool.size
94
+ needed = @config[:concurrency] + 2
68
95
  raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
69
96
 
70
97
  # cache process identity
71
- Sidekiq.options[:identity] = identity
98
+ @config[:identity] = identity
72
99
 
73
100
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
74
- Sidekiq.server_middleware
101
+ @config.server_middleware
75
102
 
76
103
  # Before this point, the process is initializing with just the main thread.
77
104
  # Starting here the process will now have multiple threads running.
78
105
  fire_event(:startup, reverse: false, reraise: true)
79
106
 
80
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
81
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
107
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
108
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
82
109
 
83
110
  launch(self_read)
84
111
  end
@@ -88,13 +115,13 @@ module Sidekiq
88
115
  logger.info "Starting processing, hit Ctrl-C to stop"
89
116
  end
90
117
 
91
- @launcher = Sidekiq::Launcher.new(options)
118
+ @launcher = Sidekiq::Launcher.new(@config)
92
119
 
93
120
  begin
94
121
  launcher.run
95
122
 
96
- while (readable_io = IO.select([self_read]))
97
- signal = readable_io.first[0].gets.strip
123
+ while self_read.wait_readable
124
+ signal = self_read.gets.strip
98
125
  handle_signal(signal)
99
126
  end
100
127
  rescue Interrupt
@@ -151,25 +178,25 @@ module Sidekiq
151
178
  # Heroku sends TERM and then waits 30 seconds for process to exit.
152
179
  "TERM" => ->(cli) { raise Interrupt },
153
180
  "TSTP" => ->(cli) {
154
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
181
+ cli.logger.info "Received TSTP, no longer accepting new work"
155
182
  cli.launcher.quiet
156
183
  },
157
184
  "TTIN" => ->(cli) {
158
185
  Thread.list.each do |thread|
159
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
186
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
160
187
  if thread.backtrace
161
- Sidekiq.logger.warn thread.backtrace.join("\n")
188
+ cli.logger.warn thread.backtrace.join("\n")
162
189
  else
163
- Sidekiq.logger.warn "<no backtrace available>"
190
+ cli.logger.warn "<no backtrace available>"
164
191
  end
165
192
  end
166
193
  }
167
194
  }
168
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
195
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
169
196
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
170
197
 
171
198
  def handle_signal(sig)
172
- Sidekiq.logger.debug "Got #{sig} signal"
199
+ logger.debug "Got #{sig} signal"
173
200
  SIGNAL_HANDLERS[sig].call(self)
174
201
  end
175
202
 
@@ -215,7 +242,7 @@ module Sidekiq
215
242
  config_dir = if File.directory?(opts[:require].to_s)
216
243
  File.join(opts[:require], "config")
217
244
  else
218
- File.join(options[:require], "config")
245
+ File.join(@config[:require], "config")
219
246
  end
220
247
 
221
248
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -228,32 +255,27 @@ module Sidekiq
228
255
  opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
229
256
 
230
257
  # set defaults
231
- opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
232
- opts[:strict] = true if opts[:strict].nil?
258
+ opts[:queues] = ["default"] if opts[:queues].nil?
233
259
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
234
260
 
235
261
  # merge with defaults
236
- options.merge!(opts)
262
+ @config.merge!(opts)
237
263
  end
238
264
 
239
- def options
240
- Sidekiq.options
241
- end
242
-
243
- def boot_system
265
+ def boot_application
244
266
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
245
267
 
246
- if File.directory?(options[:require])
268
+ if File.directory?(@config[:require])
247
269
  require "rails"
248
270
  if ::Rails::VERSION::MAJOR < 5
249
271
  raise "Sidekiq no longer supports this version of Rails"
250
272
  else
251
273
  require "sidekiq/rails"
252
- require File.expand_path("#{options[:require]}/config/environment.rb")
274
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
253
275
  end
254
- options[:tag] ||= default_tag
276
+ @config[:tag] ||= default_tag
255
277
  else
256
- require options[:require]
278
+ require @config[:require]
257
279
  end
258
280
  end
259
281
 
@@ -270,18 +292,18 @@ module Sidekiq
270
292
  end
271
293
 
272
294
  def validate!
273
- if !File.exist?(options[:require]) ||
274
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
295
+ if !File.exist?(@config[:require]) ||
296
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
275
297
  logger.info "=================================================================="
276
298
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
277
- logger.info " to load your worker classes with -r [DIR|FILE]."
299
+ logger.info " to load your job classes with -r [DIR|FILE]."
278
300
  logger.info "=================================================================="
279
301
  logger.info @parser
280
302
  die(1)
281
303
  end
282
304
 
283
305
  [:concurrency, :timeout].each do |opt|
284
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
306
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
285
307
  end
286
308
  end
287
309
 
@@ -315,7 +337,7 @@ module Sidekiq
315
337
  parse_queue opts, queue, weight
316
338
  end
317
339
 
318
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
340
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
319
341
  opts[:require] = arg
320
342
  end
321
343
 
@@ -355,11 +377,13 @@ module Sidekiq
355
377
  end
356
378
 
357
379
  def initialize_logger
358
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
380
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
359
381
  end
360
382
 
361
383
  def parse_config(path)
362
- opts = YAML.load(ERB.new(File.read(path)).result) || {}
384
+ erb = ERB.new(File.read(path))
385
+ erb.filename = File.expand_path(path)
386
+ opts = load_yaml(erb.result) || {}
363
387
 
364
388
  if opts.respond_to? :deep_symbolize_keys!
365
389
  opts.deep_symbolize_keys!
@@ -368,19 +392,30 @@ module Sidekiq
368
392
  end
369
393
 
370
394
  opts = opts.merge(opts.delete(environment.to_sym) || {})
395
+ opts.delete(:strict)
396
+
371
397
  parse_queues(opts, opts.delete(:queues) || [])
372
398
 
373
399
  opts
374
400
  end
375
401
 
402
+ def load_yaml(src)
403
+ if Psych::VERSION > "4.0"
404
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
405
+ else
406
+ YAML.load(src)
407
+ end
408
+ end
409
+
376
410
  def parse_queues(opts, queues_and_weights)
377
411
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
378
412
  end
379
413
 
380
414
  def parse_queue(opts, queue, weight = nil)
381
415
  opts[:queues] ||= []
416
+ opts[:strict] = true if opts[:strict].nil?
382
417
  raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
383
- [weight.to_i, 1].max.times { opts[:queues] << queue }
418
+ [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
384
419
  opts[:strict] = false if weight.to_i > 0
385
420
  end
386
421
 
@@ -2,9 +2,12 @@
2
2
 
3
3
  require "securerandom"
4
4
  require "sidekiq/middleware/chain"
5
+ require "sidekiq/job_util"
5
6
 
6
7
  module Sidekiq
7
8
  class Client
9
+ include Sidekiq::JobUtil
10
+
8
11
  ##
9
12
  # Define client-side middleware:
10
13
  #
@@ -12,14 +15,14 @@ module Sidekiq
12
15
  # client.middleware do |chain|
13
16
  # chain.use MyClientMiddleware
14
17
  # end
15
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
16
19
  #
17
20
  # All client instances default to the globally-defined
18
21
  # Sidekiq.client_middleware but you can change as necessary.
19
22
  #
20
23
  def middleware(&block)
21
24
  @chain ||= Sidekiq.client_middleware
22
- if block_given?
25
+ if block
23
26
  @chain = @chain.dup
24
27
  yield @chain
25
28
  end
@@ -46,16 +49,16 @@ module Sidekiq
46
49
  # The main method used to push a job to Redis. Accepts a number of options:
47
50
  #
48
51
  # queue - the named queue to use, default 'default'
49
- # class - the worker class to call, required
52
+ # class - the job class to call, required
50
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
51
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
52
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
53
56
  # backtrace - whether to save any error backtrace, default false
54
57
  #
55
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
56
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
57
60
  #
58
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
59
62
  #
60
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
61
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +67,15 @@ module Sidekiq
64
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
65
68
  #
66
69
  # Example:
67
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
68
71
  #
69
72
  def push(item)
70
73
  normed = normalize_item(item)
71
- payload = process_single(item["class"], normed)
72
-
74
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
73
77
  if payload
78
+ verify_json(payload)
74
79
  raw_push([payload])
75
80
  payload["jid"]
76
81
  end
@@ -90,19 +95,25 @@ module Sidekiq
90
95
  # Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
91
96
  # than the number given if the middleware stopped processing for one or more jobs.
92
97
  def push_bulk(items)
93
- arg = items["args"].first
94
- return [] unless arg # no jobs to push
95
- raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
98
+ args = items["args"]
99
+ raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
100
+ return [] if args.empty? # no jobs to push
96
101
 
97
102
  at = items.delete("at")
98
- raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
103
+ raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
104
+ raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
105
+
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
99
108
 
100
109
  normed = normalize_item(items)
101
- payloads = items["args"].map.with_index { |args, index|
102
- copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
110
+ payloads = args.map.with_index { |job_args, index|
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
103
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
104
-
105
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
106
117
  result || nil
107
118
  }.compact
108
119
 
@@ -115,8 +126,8 @@ module Sidekiq
115
126
  #
116
127
  # pool = ConnectionPool.new { Redis.new }
117
128
  # Sidekiq::Client.via(pool) do
118
- # SomeWorker.perform_async(1,2,3)
119
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
120
131
  # end
121
132
  #
122
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -141,10 +152,10 @@ module Sidekiq
141
152
  end
142
153
 
143
154
  # Resque compatibility helpers. Note all helpers
144
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
145
156
  #
146
157
  # Example usage:
147
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
148
159
  #
149
160
  # Messages are enqueued to the 'default' queue.
150
161
  #
@@ -153,14 +164,14 @@ module Sidekiq
153
164
  end
154
165
 
155
166
  # Example usage:
156
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
157
168
  #
158
169
  def enqueue_to(queue, klass, *args)
159
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
160
171
  end
161
172
 
162
173
  # Example usage:
163
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
164
175
  #
165
176
  def enqueue_to_in(queue, interval, klass, *args)
166
177
  int = interval.to_f
@@ -174,7 +185,7 @@ module Sidekiq
174
185
  end
175
186
 
176
187
  # Example usage:
177
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
178
189
  #
179
190
  def enqueue_in(interval, klass, *args)
180
191
  klass.perform_in(interval, *args)
@@ -185,8 +196,23 @@ module Sidekiq
185
196
 
186
197
  def raw_push(payloads)
187
198
  @redis_pool.with do |conn|
188
- conn.multi do
189
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue RedisConnection.adapter::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
190
216
  end
191
217
  end
192
218
  true
@@ -194,7 +220,7 @@ module Sidekiq
194
220
 
195
221
  def atomic_push(conn, payloads)
196
222
  if payloads.first.key?("at")
197
- conn.zadd("schedule", payloads.map { |hash|
223
+ conn.zadd("schedule", *payloads.map { |hash|
198
224
  at = hash.delete("at").to_s
199
225
  [at, Sidekiq.dump_json(hash)]
200
226
  })
@@ -209,50 +235,5 @@ module Sidekiq
209
235
  conn.lpush("queue:#{queue}", to_push)
210
236
  end
211
237
  end
212
-
213
- def process_single(worker_class, item)
214
- queue = item["queue"]
215
-
216
- middleware.invoke(worker_class, item, queue, @redis_pool) do
217
- item
218
- end
219
- end
220
-
221
- def normalize_item(item)
222
- # 6.0.0 push_bulk bug, #4321
223
- # TODO Remove after a while...
224
- item.delete("at") if item.key?("at") && item["at"].nil?
225
-
226
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
227
- raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
228
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
229
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
230
- raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
231
- # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
232
-
233
- # merge in the default sidekiq_options for the item's class and/or wrapped element
234
- # this allows ActiveJobs to control sidekiq_options too.
235
- defaults = normalized_hash(item["class"])
236
- defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
237
- item = defaults.merge(item)
238
-
239
- raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
240
-
241
- item["class"] = item["class"].to_s
242
- item["queue"] = item["queue"].to_s
243
- item["jid"] ||= SecureRandom.hex(12)
244
- item["created_at"] ||= Time.now.to_f
245
-
246
- item
247
- end
248
-
249
- def normalized_hash(item_class)
250
- if item_class.is_a?(Class)
251
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
252
- item_class.get_sidekiq_options
253
- else
254
- Sidekiq.default_worker_options
255
- end
256
- end
257
238
  end
258
239
  end
@@ -1,15 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
- require "socket"
4
- require "securerandom"
5
- require "sidekiq/exception_handler"
6
-
7
1
  module Sidekiq
8
2
  ##
9
- # This module is part of Sidekiq core and not intended for extensions.
10
- #
11
- module Util
12
- include ExceptionHandler
3
+ # Sidekiq::Component assumes a config instance is available at @config
4
+ module Component
5
+ attr_reader :config
13
6
 
14
7
  def watchdog(last_words)
15
8
  yield
@@ -26,11 +19,11 @@ module Sidekiq
26
19
  end
27
20
 
28
21
  def logger
29
- Sidekiq.logger
22
+ config.logger
30
23
  end
31
24
 
32
25
  def redis(&block)
33
- Sidekiq.redis(&block)
26
+ config.redis(&block)
34
27
  end
35
28
 
36
29
  def tid
@@ -49,11 +42,15 @@ module Sidekiq
49
42
  @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
50
43
  end
51
44
 
45
+ def handle_exception(ex, ctx = {})
46
+ config.handle_exception(ex, ctx)
47
+ end
48
+
52
49
  def fire_event(event, options = {})
53
50
  reverse = options[:reverse]
54
51
  reraise = options[:reraise]
55
52
 
56
- arr = Sidekiq.options[:lifecycle_events][event]
53
+ arr = config[:lifecycle_events][event]
57
54
  arr.reverse! if reverse
58
55
  arr.each do |block|
59
56
  block.call
@@ -61,7 +58,7 @@ module Sidekiq
61
58
  handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
62
59
  raise ex if reraise
63
60
  end
64
- arr.clear
61
+ arr.clear # once we've fired an event, we never fire it again
65
62
  end
66
63
  end
67
64
  end
data/lib/sidekiq/delay.rb CHANGED
@@ -3,6 +3,8 @@
3
3
  module Sidekiq
4
4
  module Extensions
5
5
  def self.enable_delay!
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
+
6
8
  if defined?(::ActiveSupport)
7
9
  require "sidekiq/extensions/active_record"
8
10
  require "sidekiq/extensions/action_mailer"
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
9
- # delivery to Sidekiq. Example:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
9
+ # delivery to Sidekiq.
10
10
  #
11
+ # @example
11
12
  # UserMailer.delay.send_welcome_email(new_user)
12
13
  # UserMailer.delay_for(5.days).send_welcome_email(new_user)
13
14
  # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
9
- # execution to Sidekiq. Examples:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
9
+ # execution to Sidekiq.
10
10
  #
11
- # User.recent_signups.each { |user| user.delay.mark_as_awesome }
11
+ # @example
12
+ # User.recent_signups.each { |user| user.delay.mark_as_awesome }
12
13
  #
13
14
  # Please note, this is not recommended as this will serialize the entire
14
15
  # object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
9
- # execution to Sidekiq. Examples:
8
+ # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
9
+ # execution to Sidekiq.
10
10
  #
11
- # User.delay.delete_inactive
12
- # Wikipedia.delay.download_changes_for(Date.today)
11
+ # @example
12
+ # User.delay.delete_inactive
13
+ # Wikipedia.delay.download_changes_for(Date.today)
13
14
  #
14
15
  class DelayedClass
15
16
  include Sidekiq::Worker
@@ -10,7 +10,7 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
16
  def method_missing(name, *args)
@@ -24,7 +24,9 @@ module Sidekiq
24
24
  if marshalled.size > SIZE_LIMIT
25
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
26
26
  end
27
- @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
27
+ @performable.client_push({"class" => @performable,
28
+ "args" => [marshalled],
29
+ "display_class" => "#{@target}.#{name}"}.merge(@opts))
28
30
  end
29
31
  end
30
32
  end