sidekiq 6.1.1 → 6.5.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +250 -3
  3. data/LICENSE +3 -3
  4. data/README.md +10 -6
  5. data/bin/sidekiq +3 -3
  6. data/bin/sidekiqload +70 -66
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +352 -156
  13. data/lib/sidekiq/cli.rb +86 -41
  14. data/lib/sidekiq/client.rb +49 -73
  15. data/lib/sidekiq/{util.rb → component.rb} +12 -14
  16. data/lib/sidekiq/delay.rb +3 -1
  17. data/lib/sidekiq/extensions/action_mailer.rb +3 -2
  18. data/lib/sidekiq/extensions/active_record.rb +1 -1
  19. data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
  20. data/lib/sidekiq/fetch.rb +31 -20
  21. data/lib/sidekiq/job.rb +13 -0
  22. data/lib/sidekiq/job_logger.rb +16 -28
  23. data/lib/sidekiq/job_retry.rb +79 -59
  24. data/lib/sidekiq/job_util.rb +71 -0
  25. data/lib/sidekiq/launcher.rb +126 -65
  26. data/lib/sidekiq/logger.rb +11 -20
  27. data/lib/sidekiq/manager.rb +35 -34
  28. data/lib/sidekiq/metrics/deploy.rb +47 -0
  29. data/lib/sidekiq/metrics/query.rb +153 -0
  30. data/lib/sidekiq/metrics/shared.rb +94 -0
  31. data/lib/sidekiq/metrics/tracking.rb +134 -0
  32. data/lib/sidekiq/middleware/chain.rb +88 -42
  33. data/lib/sidekiq/middleware/current_attributes.rb +63 -0
  34. data/lib/sidekiq/middleware/i18n.rb +6 -4
  35. data/lib/sidekiq/middleware/modules.rb +21 -0
  36. data/lib/sidekiq/monitor.rb +2 -2
  37. data/lib/sidekiq/paginator.rb +17 -9
  38. data/lib/sidekiq/processor.rb +47 -41
  39. data/lib/sidekiq/rails.rb +32 -4
  40. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  41. data/lib/sidekiq/redis_connection.rb +84 -55
  42. data/lib/sidekiq/ring_buffer.rb +29 -0
  43. data/lib/sidekiq/scheduled.rb +96 -32
  44. data/lib/sidekiq/testing/inline.rb +4 -4
  45. data/lib/sidekiq/testing.rb +38 -39
  46. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  47. data/lib/sidekiq/version.rb +1 -1
  48. data/lib/sidekiq/web/action.rb +3 -3
  49. data/lib/sidekiq/web/application.rb +41 -16
  50. data/lib/sidekiq/web/csrf_protection.rb +32 -5
  51. data/lib/sidekiq/web/helpers.rb +52 -30
  52. data/lib/sidekiq/web/router.rb +4 -1
  53. data/lib/sidekiq/web.rb +38 -78
  54. data/lib/sidekiq/worker.rb +142 -16
  55. data/lib/sidekiq.rb +114 -31
  56. data/sidekiq.gemspec +12 -4
  57. data/web/assets/images/apple-touch-icon.png +0 -0
  58. data/web/assets/javascripts/application.js +114 -60
  59. data/web/assets/javascripts/chart.min.js +13 -0
  60. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  61. data/web/assets/javascripts/dashboard.js +50 -67
  62. data/web/assets/javascripts/graph.js +16 -0
  63. data/web/assets/javascripts/metrics.js +262 -0
  64. data/web/assets/stylesheets/application-dark.css +61 -51
  65. data/web/assets/stylesheets/application-rtl.css +0 -4
  66. data/web/assets/stylesheets/application.css +84 -243
  67. data/web/locales/ar.yml +8 -2
  68. data/web/locales/el.yml +43 -19
  69. data/web/locales/en.yml +11 -1
  70. data/web/locales/es.yml +18 -2
  71. data/web/locales/fr.yml +8 -1
  72. data/web/locales/ja.yml +10 -0
  73. data/web/locales/lt.yml +1 -1
  74. data/web/locales/pt-br.yml +27 -9
  75. data/web/locales/ru.yml +4 -0
  76. data/web/locales/zh-cn.yml +36 -11
  77. data/web/locales/zh-tw.yml +32 -7
  78. data/web/views/_footer.erb +1 -1
  79. data/web/views/_job_info.erb +1 -1
  80. data/web/views/_nav.erb +1 -1
  81. data/web/views/_poll_link.erb +2 -5
  82. data/web/views/_summary.erb +7 -7
  83. data/web/views/busy.erb +57 -21
  84. data/web/views/dashboard.erb +23 -14
  85. data/web/views/dead.erb +1 -1
  86. data/web/views/layout.erb +2 -1
  87. data/web/views/metrics.erb +69 -0
  88. data/web/views/metrics_for_job.erb +87 -0
  89. data/web/views/morgue.erb +6 -6
  90. data/web/views/queue.erb +15 -11
  91. data/web/views/queues.erb +4 -4
  92. data/web/views/retries.erb +7 -7
  93. data/web/views/retry.erb +1 -1
  94. data/web/views/scheduled.erb +1 -1
  95. metadata +52 -39
  96. data/.circleci/config.yml +0 -71
  97. data/.github/contributing.md +0 -32
  98. data/.github/issue_template.md +0 -11
  99. data/.gitignore +0 -13
  100. data/.standard.yml +0 -20
  101. data/3.0-Upgrade.md +0 -70
  102. data/4.0-Upgrade.md +0 -53
  103. data/5.0-Upgrade.md +0 -56
  104. data/6.0-Upgrade.md +0 -72
  105. data/COMM-LICENSE +0 -97
  106. data/Ent-2.0-Upgrade.md +0 -37
  107. data/Ent-Changes.md +0 -275
  108. data/Gemfile +0 -24
  109. data/Gemfile.lock +0 -208
  110. data/Pro-2.0-Upgrade.md +0 -138
  111. data/Pro-3.0-Upgrade.md +0 -44
  112. data/Pro-4.0-Upgrade.md +0 -35
  113. data/Pro-5.0-Upgrade.md +0 -25
  114. data/Pro-Changes.md +0 -795
  115. data/Rakefile +0 -10
  116. data/code_of_conduct.md +0 -50
  117. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  118. data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,34 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/component"
12
13
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
14
 
15
- module Sidekiq
15
+ # module ScoutApm
16
+ # VERSION = "5.3.1"
17
+ # end
18
+ fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
+
20
+
21
+ scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
+
23
+
24
+ EOM
25
+
26
+ module Sidekiq # :nodoc:
16
27
  class CLI
17
- include Util
28
+ include Sidekiq::Component
18
29
  include Singleton unless $TESTING
19
30
 
20
31
  attr_accessor :launcher
21
32
  attr_accessor :environment
33
+ attr_accessor :config
34
+
35
+ def parse(args = ARGV.dup)
36
+ @config = Sidekiq
37
+ @config[:error_handlers].clear
38
+ @config[:error_handlers] << @config.method(:default_error_handler)
22
39
 
23
- def parse(args = ARGV)
24
40
  setup_options(args)
25
41
  initialize_logger
26
42
  validate!
@@ -36,7 +52,7 @@ module Sidekiq
36
52
  def run(boot_app: true)
37
53
  boot_application if boot_app
38
54
 
39
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
55
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
40
56
  print_banner
41
57
  end
42
58
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -44,9 +60,17 @@ module Sidekiq
44
60
  self_read, self_write = IO.pipe
45
61
  sigs = %w[INT TERM TTIN TSTP]
46
62
  # USR1 and USR2 don't work on the JVM
47
- sigs << "USR2" unless jruby?
63
+ sigs << "USR2" if Sidekiq.pro? && !jruby?
48
64
  sigs.each do |sig|
49
- trap sig do
65
+ old_handler = Signal.trap(sig) do
66
+ if old_handler.respond_to?(:call)
67
+ begin
68
+ old_handler.call
69
+ rescue Exception => exc
70
+ # signal handlers can't use Logger so puts only
71
+ puts ["Error in #{sig} handler", exc].inspect
72
+ end
73
+ end
50
74
  self_write.puts(sig)
51
75
  end
52
76
  rescue ArgumentError
@@ -59,27 +83,40 @@ module Sidekiq
59
83
 
60
84
  # touch the connection pool so it is created before we
61
85
  # fire startup and start multithreading.
62
- ver = Sidekiq.redis_info["redis_version"]
86
+ info = @config.redis_info
87
+ ver = info["redis_version"]
63
88
  raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
64
89
 
90
+ maxmemory_policy = info["maxmemory_policy"]
91
+ if maxmemory_policy != "noeviction"
92
+ logger.warn <<~EOM
93
+
94
+
95
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
96
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
97
+ See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
98
+
99
+ EOM
100
+ end
101
+
65
102
  # Since the user can pass us a connection pool explicitly in the initializer, we
66
103
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
67
- cursize = Sidekiq.redis_pool.size
68
- needed = Sidekiq.options[:concurrency] + 2
104
+ cursize = @config.redis_pool.size
105
+ needed = @config[:concurrency] + 2
69
106
  raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
70
107
 
71
108
  # cache process identity
72
- Sidekiq.options[:identity] = identity
109
+ @config[:identity] = identity
73
110
 
74
111
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
75
- Sidekiq.server_middleware
112
+ @config.server_middleware
76
113
 
77
114
  # Before this point, the process is initializing with just the main thread.
78
115
  # Starting here the process will now have multiple threads running.
79
116
  fire_event(:startup, reverse: false, reraise: true)
80
117
 
81
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
82
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
118
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
119
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
83
120
 
84
121
  launch(self_read)
85
122
  end
@@ -89,13 +126,13 @@ module Sidekiq
89
126
  logger.info "Starting processing, hit Ctrl-C to stop"
90
127
  end
91
128
 
92
- @launcher = Sidekiq::Launcher.new(options)
129
+ @launcher = Sidekiq::Launcher.new(@config)
93
130
 
94
131
  begin
95
132
  launcher.run
96
133
 
97
- while (readable_io = IO.select([self_read]))
98
- signal = readable_io.first[0].gets.strip
134
+ while self_read.wait_readable
135
+ signal = self_read.gets.strip
99
136
  handle_signal(signal)
100
137
  end
101
138
  rescue Interrupt
@@ -152,25 +189,25 @@ module Sidekiq
152
189
  # Heroku sends TERM and then waits 30 seconds for process to exit.
153
190
  "TERM" => ->(cli) { raise Interrupt },
154
191
  "TSTP" => ->(cli) {
155
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
192
+ cli.logger.info "Received TSTP, no longer accepting new work"
156
193
  cli.launcher.quiet
157
194
  },
158
195
  "TTIN" => ->(cli) {
159
196
  Thread.list.each do |thread|
160
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
197
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
161
198
  if thread.backtrace
162
- Sidekiq.logger.warn thread.backtrace.join("\n")
199
+ cli.logger.warn thread.backtrace.join("\n")
163
200
  else
164
- Sidekiq.logger.warn "<no backtrace available>"
201
+ cli.logger.warn "<no backtrace available>"
165
202
  end
166
203
  end
167
204
  }
168
205
  }
169
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
206
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
170
207
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
171
208
 
172
209
  def handle_signal(sig)
173
- Sidekiq.logger.debug "Got #{sig} signal"
210
+ logger.debug "Got #{sig} signal"
174
211
  SIGNAL_HANDLERS[sig].call(self)
175
212
  end
176
213
 
@@ -188,6 +225,7 @@ module Sidekiq
188
225
  # Both Sinatra 2.0+ and Sidekiq support this term.
189
226
  # RAILS_ENV and RACK_ENV are there for legacy support.
190
227
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
+ config[:environment] = @environment
191
229
  end
192
230
 
193
231
  def symbolize_keys_deep!(hash)
@@ -216,7 +254,7 @@ module Sidekiq
216
254
  config_dir = if File.directory?(opts[:require].to_s)
217
255
  File.join(opts[:require], "config")
218
256
  else
219
- File.join(options[:require], "config")
257
+ File.join(@config[:require], "config")
220
258
  end
221
259
 
222
260
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -233,27 +271,23 @@ module Sidekiq
233
271
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
234
272
 
235
273
  # merge with defaults
236
- options.merge!(opts)
237
- end
238
-
239
- def options
240
- Sidekiq.options
274
+ @config.merge!(opts)
241
275
  end
242
276
 
243
277
  def boot_application
244
278
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
245
279
 
246
- if File.directory?(options[:require])
280
+ if File.directory?(@config[:require])
247
281
  require "rails"
248
282
  if ::Rails::VERSION::MAJOR < 5
249
283
  raise "Sidekiq no longer supports this version of Rails"
250
284
  else
251
285
  require "sidekiq/rails"
252
- require File.expand_path("#{options[:require]}/config/environment.rb")
286
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
253
287
  end
254
- options[:tag] ||= default_tag
288
+ @config[:tag] ||= default_tag
255
289
  else
256
- require options[:require]
290
+ require @config[:require]
257
291
  end
258
292
  end
259
293
 
@@ -270,18 +304,18 @@ module Sidekiq
270
304
  end
271
305
 
272
306
  def validate!
273
- if !File.exist?(options[:require]) ||
274
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
307
+ if !File.exist?(@config[:require]) ||
308
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
275
309
  logger.info "=================================================================="
276
310
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
277
- logger.info " to load your worker classes with -r [DIR|FILE]."
311
+ logger.info " to load your job classes with -r [DIR|FILE]."
278
312
  logger.info "=================================================================="
279
313
  logger.info @parser
280
314
  die(1)
281
315
  end
282
316
 
283
317
  [:concurrency, :timeout].each do |opt|
284
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
318
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
285
319
  end
286
320
  end
287
321
 
@@ -315,7 +349,7 @@ module Sidekiq
315
349
  parse_queue opts, queue, weight
316
350
  end
317
351
 
318
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
352
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
319
353
  opts[:require] = arg
320
354
  end
321
355
 
@@ -355,11 +389,13 @@ module Sidekiq
355
389
  end
356
390
 
357
391
  def initialize_logger
358
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
392
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
359
393
  end
360
394
 
361
395
  def parse_config(path)
362
- opts = YAML.load(ERB.new(File.read(path)).result) || {}
396
+ erb = ERB.new(File.read(path))
397
+ erb.filename = File.expand_path(path)
398
+ opts = load_yaml(erb.result) || {}
363
399
 
364
400
  if opts.respond_to? :deep_symbolize_keys!
365
401
  opts.deep_symbolize_keys!
@@ -375,6 +411,14 @@ module Sidekiq
375
411
  opts
376
412
  end
377
413
 
414
+ def load_yaml(src)
415
+ if Psych::VERSION > "4.0"
416
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
417
+ else
418
+ YAML.load(src)
419
+ end
420
+ end
421
+
378
422
  def parse_queues(opts, queues_and_weights)
379
423
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
380
424
  end
@@ -383,7 +427,7 @@ module Sidekiq
383
427
  opts[:queues] ||= []
384
428
  opts[:strict] = true if opts[:strict].nil?
385
429
  raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
386
- [weight.to_i, 1].max.times { opts[:queues] << queue }
430
+ [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
387
431
  opts[:strict] = false if weight.to_i > 0
388
432
  end
389
433
 
@@ -394,3 +438,4 @@ module Sidekiq
394
438
  end
395
439
 
396
440
  require "sidekiq/systemd"
441
+ require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
@@ -2,9 +2,12 @@
2
2
 
3
3
  require "securerandom"
4
4
  require "sidekiq/middleware/chain"
5
+ require "sidekiq/job_util"
5
6
 
6
7
  module Sidekiq
7
8
  class Client
9
+ include Sidekiq::JobUtil
10
+
8
11
  ##
9
12
  # Define client-side middleware:
10
13
  #
@@ -12,14 +15,14 @@ module Sidekiq
12
15
  # client.middleware do |chain|
13
16
  # chain.use MyClientMiddleware
14
17
  # end
15
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
16
19
  #
17
20
  # All client instances default to the globally-defined
18
21
  # Sidekiq.client_middleware but you can change as necessary.
19
22
  #
20
23
  def middleware(&block)
21
24
  @chain ||= Sidekiq.client_middleware
22
- if block_given?
25
+ if block
23
26
  @chain = @chain.dup
24
27
  yield @chain
25
28
  end
@@ -46,16 +49,16 @@ module Sidekiq
46
49
  # The main method used to push a job to Redis. Accepts a number of options:
47
50
  #
48
51
  # queue - the named queue to use, default 'default'
49
- # class - the worker class to call, required
52
+ # class - the job class to call, required
50
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
51
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
52
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
53
56
  # backtrace - whether to save any error backtrace, default false
54
57
  #
55
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
56
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
57
60
  #
58
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
59
62
  #
60
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
61
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +67,15 @@ module Sidekiq
64
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
65
68
  #
66
69
  # Example:
67
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
68
71
  #
69
72
  def push(item)
70
73
  normed = normalize_item(item)
71
- payload = process_single(item["class"], normed)
72
-
74
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
73
77
  if payload
78
+ verify_json(payload)
74
79
  raw_push([payload])
75
80
  payload["jid"]
76
81
  end
@@ -95,15 +100,20 @@ module Sidekiq
95
100
  return [] if args.empty? # no jobs to push
96
101
 
97
102
  at = items.delete("at")
98
- raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
103
+ raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
99
104
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
100
105
 
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
108
+
101
109
  normed = normalize_item(items)
102
110
  payloads = args.map.with_index { |job_args, index|
103
- copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
104
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
105
-
106
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
107
117
  result || nil
108
118
  }.compact
109
119
 
@@ -116,8 +126,8 @@ module Sidekiq
116
126
  #
117
127
  # pool = ConnectionPool.new { Redis.new }
118
128
  # Sidekiq::Client.via(pool) do
119
- # SomeWorker.perform_async(1,2,3)
120
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
121
131
  # end
122
132
  #
123
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -142,10 +152,10 @@ module Sidekiq
142
152
  end
143
153
 
144
154
  # Resque compatibility helpers. Note all helpers
145
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
146
156
  #
147
157
  # Example usage:
148
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
149
159
  #
150
160
  # Messages are enqueued to the 'default' queue.
151
161
  #
@@ -154,19 +164,19 @@ module Sidekiq
154
164
  end
155
165
 
156
166
  # Example usage:
157
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
158
168
  #
159
169
  def enqueue_to(queue, klass, *args)
160
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
161
171
  end
162
172
 
163
173
  # Example usage:
164
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
165
175
  #
166
176
  def enqueue_to_in(queue, interval, klass, *args)
167
177
  int = interval.to_f
168
178
  now = Time.now.to_f
169
- ts = (int < 1_000_000_000 ? now + int : int)
179
+ ts = ((int < 1_000_000_000) ? now + int : int)
170
180
 
171
181
  item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
172
182
  item.delete("at") if ts <= now
@@ -175,7 +185,7 @@ module Sidekiq
175
185
  end
176
186
 
177
187
  # Example usage:
178
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
179
189
  #
180
190
  def enqueue_in(interval, klass, *args)
181
191
  klass.perform_in(interval, *args)
@@ -186,8 +196,23 @@ module Sidekiq
186
196
 
187
197
  def raw_push(payloads)
188
198
  @redis_pool.with do |conn|
189
- conn.multi do
190
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue RedisConnection.adapter::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
191
216
  end
192
217
  end
193
218
  true
@@ -195,7 +220,7 @@ module Sidekiq
195
220
 
196
221
  def atomic_push(conn, payloads)
197
222
  if payloads.first.key?("at")
198
- conn.zadd("schedule", payloads.map { |hash|
223
+ conn.zadd("schedule", payloads.flat_map { |hash|
199
224
  at = hash.delete("at").to_s
200
225
  [at, Sidekiq.dump_json(hash)]
201
226
  })
@@ -206,58 +231,9 @@ module Sidekiq
206
231
  entry["enqueued_at"] = now
207
232
  Sidekiq.dump_json(entry)
208
233
  }
209
- conn.sadd("queues", queue)
234
+ conn.sadd("queues", [queue])
210
235
  conn.lpush("queue:#{queue}", to_push)
211
236
  end
212
237
  end
213
-
214
- def process_single(worker_class, item)
215
- queue = item["queue"]
216
-
217
- middleware.invoke(worker_class, item, queue, @redis_pool) do
218
- item
219
- end
220
- end
221
-
222
- def validate(item)
223
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
224
- raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
225
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
226
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
227
- raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
228
- end
229
-
230
- def normalize_item(item)
231
- # 6.0.0 push_bulk bug, #4321
232
- # TODO Remove after a while...
233
- item.delete("at") if item.key?("at") && item["at"].nil?
234
-
235
- validate(item)
236
- # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
237
-
238
- # merge in the default sidekiq_options for the item's class and/or wrapped element
239
- # this allows ActiveJobs to control sidekiq_options too.
240
- defaults = normalized_hash(item["class"])
241
- defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
242
- item = defaults.merge(item)
243
-
244
- raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
245
-
246
- item["class"] = item["class"].to_s
247
- item["queue"] = item["queue"].to_s
248
- item["jid"] ||= SecureRandom.hex(12)
249
- item["created_at"] ||= Time.now.to_f
250
-
251
- item
252
- end
253
-
254
- def normalized_hash(item_class)
255
- if item_class.is_a?(Class)
256
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
257
- item_class.get_sidekiq_options
258
- else
259
- Sidekiq.default_worker_options
260
- end
261
- end
262
238
  end
263
239
  end
@@ -1,15 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
- require "socket"
4
- require "securerandom"
5
- require "sidekiq/exception_handler"
6
-
7
1
  module Sidekiq
8
2
  ##
9
- # This module is part of Sidekiq core and not intended for extensions.
10
- #
11
- module Util
12
- include ExceptionHandler
3
+ # Sidekiq::Component assumes a config instance is available at @config
4
+ module Component # :nodoc:
5
+ attr_reader :config
13
6
 
14
7
  def watchdog(last_words)
15
8
  yield
@@ -26,11 +19,11 @@ module Sidekiq
26
19
  end
27
20
 
28
21
  def logger
29
- Sidekiq.logger
22
+ config.logger
30
23
  end
31
24
 
32
25
  def redis(&block)
33
- Sidekiq.redis(&block)
26
+ config.redis(&block)
34
27
  end
35
28
 
36
29
  def tid
@@ -49,11 +42,16 @@ module Sidekiq
49
42
  @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
50
43
  end
51
44
 
45
+ def handle_exception(ex, ctx = {})
46
+ config.handle_exception(ex, ctx)
47
+ end
48
+
52
49
  def fire_event(event, options = {})
50
+ oneshot = options.fetch(:oneshot, true)
53
51
  reverse = options[:reverse]
54
52
  reraise = options[:reraise]
55
53
 
56
- arr = Sidekiq.options[:lifecycle_events][event]
54
+ arr = config[:lifecycle_events][event]
57
55
  arr.reverse! if reverse
58
56
  arr.each do |block|
59
57
  block.call
@@ -61,7 +59,7 @@ module Sidekiq
61
59
  handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
62
60
  raise ex if reraise
63
61
  end
64
- arr.clear
62
+ arr.clear if oneshot # once we've fired an event, we never fire it again
65
63
  end
66
64
  end
67
65
  end
data/lib/sidekiq/delay.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Sidekiq
3
+ module Sidekiq # :nodoc:
4
4
  module Extensions
5
5
  def self.enable_delay!
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
+
6
8
  if defined?(::ActiveSupport)
7
9
  require "sidekiq/extensions/active_record"
8
10
  require "sidekiq/extensions/action_mailer"
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
9
- # delivery to Sidekiq. Example:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
9
+ # delivery to Sidekiq.
10
10
  #
11
+ # @example
11
12
  # UserMailer.delay.send_welcome_email(new_user)
12
13
  # UserMailer.delay_for(5.days).send_welcome_email(new_user)
13
14
  # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -5,7 +5,7 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
9
9
  # execution to Sidekiq.
10
10
  #
11
11
  # @example
@@ -10,7 +10,7 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
16
  def method_missing(name, *args)
@@ -24,7 +24,9 @@ module Sidekiq
24
24
  if marshalled.size > SIZE_LIMIT
25
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
26
26
  end
27
- @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
27
+ @performable.client_push({"class" => @performable,
28
+ "args" => [marshalled],
29
+ "display_class" => "#{@target}.#{name}"}.merge(@opts))
28
30
  end
29
31
  end
30
32
  end