sidekiq 6.3.1 → 6.5.9

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +134 -0
  3. data/LICENSE +3 -3
  4. data/README.md +7 -2
  5. data/bin/sidekiq +3 -3
  6. data/bin/sidekiqload +70 -66
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +261 -104
  13. data/lib/sidekiq/cli.rb +62 -38
  14. data/lib/sidekiq/client.rb +47 -67
  15. data/lib/sidekiq/{util.rb → component.rb} +12 -42
  16. data/lib/sidekiq/delay.rb +3 -1
  17. data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
  18. data/lib/sidekiq/fetch.rb +20 -18
  19. data/lib/sidekiq/job_logger.rb +15 -27
  20. data/lib/sidekiq/job_retry.rb +78 -55
  21. data/lib/sidekiq/job_util.rb +71 -0
  22. data/lib/sidekiq/launcher.rb +58 -54
  23. data/lib/sidekiq/logger.rb +8 -18
  24. data/lib/sidekiq/manager.rb +35 -34
  25. data/lib/sidekiq/metrics/deploy.rb +47 -0
  26. data/lib/sidekiq/metrics/query.rb +153 -0
  27. data/lib/sidekiq/metrics/shared.rb +94 -0
  28. data/lib/sidekiq/metrics/tracking.rb +134 -0
  29. data/lib/sidekiq/middleware/chain.rb +82 -38
  30. data/lib/sidekiq/middleware/current_attributes.rb +19 -8
  31. data/lib/sidekiq/middleware/i18n.rb +6 -4
  32. data/lib/sidekiq/middleware/modules.rb +21 -0
  33. data/lib/sidekiq/monitor.rb +2 -2
  34. data/lib/sidekiq/paginator.rb +17 -9
  35. data/lib/sidekiq/processor.rb +47 -41
  36. data/lib/sidekiq/rails.rb +15 -8
  37. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  38. data/lib/sidekiq/redis_connection.rb +80 -49
  39. data/lib/sidekiq/ring_buffer.rb +29 -0
  40. data/lib/sidekiq/scheduled.rb +66 -27
  41. data/lib/sidekiq/testing/inline.rb +4 -4
  42. data/lib/sidekiq/testing.rb +37 -36
  43. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  44. data/lib/sidekiq/version.rb +1 -1
  45. data/lib/sidekiq/web/action.rb +3 -3
  46. data/lib/sidekiq/web/application.rb +26 -7
  47. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  48. data/lib/sidekiq/web/helpers.rb +21 -8
  49. data/lib/sidekiq/web.rb +8 -4
  50. data/lib/sidekiq/worker.rb +78 -19
  51. data/lib/sidekiq.rb +111 -30
  52. data/sidekiq.gemspec +2 -2
  53. data/web/assets/javascripts/application.js +58 -26
  54. data/web/assets/javascripts/chart.min.js +13 -0
  55. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  56. data/web/assets/javascripts/dashboard.js +0 -17
  57. data/web/assets/javascripts/graph.js +16 -0
  58. data/web/assets/javascripts/metrics.js +262 -0
  59. data/web/assets/stylesheets/application-dark.css +13 -17
  60. data/web/assets/stylesheets/application.css +48 -6
  61. data/web/locales/el.yml +43 -19
  62. data/web/locales/en.yml +7 -0
  63. data/web/locales/ja.yml +7 -0
  64. data/web/locales/pt-br.yml +27 -9
  65. data/web/locales/zh-cn.yml +36 -11
  66. data/web/locales/zh-tw.yml +32 -7
  67. data/web/views/_nav.erb +1 -1
  68. data/web/views/_summary.erb +1 -1
  69. data/web/views/busy.erb +9 -4
  70. data/web/views/dashboard.erb +1 -0
  71. data/web/views/metrics.erb +69 -0
  72. data/web/views/metrics_for_job.erb +87 -0
  73. data/web/views/queue.erb +5 -1
  74. metadata +39 -13
  75. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  76. data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,34 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/component"
12
13
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
14
 
15
- module Sidekiq
15
+ # module ScoutApm
16
+ # VERSION = "5.3.1"
17
+ # end
18
+ fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
+
20
+
21
+ scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
+
23
+
24
+ EOM
25
+
26
+ module Sidekiq # :nodoc:
16
27
  class CLI
17
- include Util
28
+ include Sidekiq::Component
18
29
  include Singleton unless $TESTING
19
30
 
20
31
  attr_accessor :launcher
21
32
  attr_accessor :environment
33
+ attr_accessor :config
34
+
35
+ def parse(args = ARGV.dup)
36
+ @config = Sidekiq
37
+ @config[:error_handlers].clear
38
+ @config[:error_handlers] << @config.method(:default_error_handler)
22
39
 
23
- def parse(args = ARGV)
24
40
  setup_options(args)
25
41
  initialize_logger
26
42
  validate!
@@ -36,7 +52,7 @@ module Sidekiq
36
52
  def run(boot_app: true)
37
53
  boot_application if boot_app
38
54
 
39
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
55
+ if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
40
56
  print_banner
41
57
  end
42
58
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +83,7 @@ module Sidekiq
67
83
 
68
84
  # touch the connection pool so it is created before we
69
85
  # fire startup and start multithreading.
70
- info = Sidekiq.redis_info
86
+ info = @config.redis_info
71
87
  ver = info["redis_version"]
72
88
  raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
73
89
 
@@ -85,22 +101,22 @@ module Sidekiq
85
101
 
86
102
  # Since the user can pass us a connection pool explicitly in the initializer, we
87
103
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
88
- cursize = Sidekiq.redis_pool.size
89
- needed = Sidekiq.options[:concurrency] + 2
104
+ cursize = @config.redis_pool.size
105
+ needed = @config[:concurrency] + 2
90
106
  raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
91
107
 
92
108
  # cache process identity
93
- Sidekiq.options[:identity] = identity
109
+ @config[:identity] = identity
94
110
 
95
111
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
96
- Sidekiq.server_middleware
112
+ @config.server_middleware
97
113
 
98
114
  # Before this point, the process is initializing with just the main thread.
99
115
  # Starting here the process will now have multiple threads running.
100
116
  fire_event(:startup, reverse: false, reraise: true)
101
117
 
102
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
103
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
118
+ logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
119
+ logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
104
120
 
105
121
  launch(self_read)
106
122
  end
@@ -110,13 +126,13 @@ module Sidekiq
110
126
  logger.info "Starting processing, hit Ctrl-C to stop"
111
127
  end
112
128
 
113
- @launcher = Sidekiq::Launcher.new(options)
129
+ @launcher = Sidekiq::Launcher.new(@config)
114
130
 
115
131
  begin
116
132
  launcher.run
117
133
 
118
- while (readable_io = IO.select([self_read]))
119
- signal = readable_io.first[0].gets.strip
134
+ while self_read.wait_readable
135
+ signal = self_read.gets.strip
120
136
  handle_signal(signal)
121
137
  end
122
138
  rescue Interrupt
@@ -173,25 +189,25 @@ module Sidekiq
173
189
  # Heroku sends TERM and then waits 30 seconds for process to exit.
174
190
  "TERM" => ->(cli) { raise Interrupt },
175
191
  "TSTP" => ->(cli) {
176
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
192
+ cli.logger.info "Received TSTP, no longer accepting new work"
177
193
  cli.launcher.quiet
178
194
  },
179
195
  "TTIN" => ->(cli) {
180
196
  Thread.list.each do |thread|
181
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
197
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
182
198
  if thread.backtrace
183
- Sidekiq.logger.warn thread.backtrace.join("\n")
199
+ cli.logger.warn thread.backtrace.join("\n")
184
200
  else
185
- Sidekiq.logger.warn "<no backtrace available>"
201
+ cli.logger.warn "<no backtrace available>"
186
202
  end
187
203
  end
188
204
  }
189
205
  }
190
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
206
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
191
207
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
192
208
 
193
209
  def handle_signal(sig)
194
- Sidekiq.logger.debug "Got #{sig} signal"
210
+ logger.debug "Got #{sig} signal"
195
211
  SIGNAL_HANDLERS[sig].call(self)
196
212
  end
197
213
 
@@ -209,6 +225,7 @@ module Sidekiq
209
225
  # Both Sinatra 2.0+ and Sidekiq support this term.
210
226
  # RAILS_ENV and RACK_ENV are there for legacy support.
211
227
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
+ config[:environment] = @environment
212
229
  end
213
230
 
214
231
  def symbolize_keys_deep!(hash)
@@ -237,7 +254,7 @@ module Sidekiq
237
254
  config_dir = if File.directory?(opts[:require].to_s)
238
255
  File.join(opts[:require], "config")
239
256
  else
240
- File.join(options[:require], "config")
257
+ File.join(@config[:require], "config")
241
258
  end
242
259
 
243
260
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +271,23 @@ module Sidekiq
254
271
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
255
272
 
256
273
  # merge with defaults
257
- options.merge!(opts)
258
- end
259
-
260
- def options
261
- Sidekiq.options
274
+ @config.merge!(opts)
262
275
  end
263
276
 
264
277
  def boot_application
265
278
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
266
279
 
267
- if File.directory?(options[:require])
280
+ if File.directory?(@config[:require])
268
281
  require "rails"
269
282
  if ::Rails::VERSION::MAJOR < 5
270
283
  raise "Sidekiq no longer supports this version of Rails"
271
284
  else
272
285
  require "sidekiq/rails"
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
286
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
274
287
  end
275
- options[:tag] ||= default_tag
288
+ @config[:tag] ||= default_tag
276
289
  else
277
- require options[:require]
290
+ require @config[:require]
278
291
  end
279
292
  end
280
293
 
@@ -291,18 +304,18 @@ module Sidekiq
291
304
  end
292
305
 
293
306
  def validate!
294
- if !File.exist?(options[:require]) ||
295
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
307
+ if !File.exist?(@config[:require]) ||
308
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
296
309
  logger.info "=================================================================="
297
310
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
298
- logger.info " to load your worker classes with -r [DIR|FILE]."
311
+ logger.info " to load your job classes with -r [DIR|FILE]."
299
312
  logger.info "=================================================================="
300
313
  logger.info @parser
301
314
  die(1)
302
315
  end
303
316
 
304
317
  [:concurrency, :timeout].each do |opt|
305
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
318
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
306
319
  end
307
320
  end
308
321
 
@@ -336,7 +349,7 @@ module Sidekiq
336
349
  parse_queue opts, queue, weight
337
350
  end
338
351
 
339
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
352
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
340
353
  opts[:require] = arg
341
354
  end
342
355
 
@@ -376,11 +389,13 @@ module Sidekiq
376
389
  end
377
390
 
378
391
  def initialize_logger
379
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
392
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
380
393
  end
381
394
 
382
395
  def parse_config(path)
383
- opts = YAML.load(ERB.new(File.read(path)).result) || {}
396
+ erb = ERB.new(File.read(path))
397
+ erb.filename = File.expand_path(path)
398
+ opts = load_yaml(erb.result) || {}
384
399
 
385
400
  if opts.respond_to? :deep_symbolize_keys!
386
401
  opts.deep_symbolize_keys!
@@ -396,6 +411,14 @@ module Sidekiq
396
411
  opts
397
412
  end
398
413
 
414
+ def load_yaml(src)
415
+ if Psych::VERSION > "4.0"
416
+ YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
417
+ else
418
+ YAML.load(src)
419
+ end
420
+ end
421
+
399
422
  def parse_queues(opts, queues_and_weights)
400
423
  queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
401
424
  end
@@ -415,3 +438,4 @@ module Sidekiq
415
438
  end
416
439
 
417
440
  require "sidekiq/systemd"
441
+ require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
@@ -2,9 +2,12 @@
2
2
 
3
3
  require "securerandom"
4
4
  require "sidekiq/middleware/chain"
5
+ require "sidekiq/job_util"
5
6
 
6
7
  module Sidekiq
7
8
  class Client
9
+ include Sidekiq::JobUtil
10
+
8
11
  ##
9
12
  # Define client-side middleware:
10
13
  #
@@ -12,7 +15,7 @@ module Sidekiq
12
15
  # client.middleware do |chain|
13
16
  # chain.use MyClientMiddleware
14
17
  # end
15
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
16
19
  #
17
20
  # All client instances default to the globally-defined
18
21
  # Sidekiq.client_middleware but you can change as necessary.
@@ -46,16 +49,16 @@ module Sidekiq
46
49
  # The main method used to push a job to Redis. Accepts a number of options:
47
50
  #
48
51
  # queue - the named queue to use, default 'default'
49
- # class - the worker class to call, required
52
+ # class - the job class to call, required
50
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
51
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
52
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
53
56
  # backtrace - whether to save any error backtrace, default false
54
57
  #
55
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
56
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
57
60
  #
58
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
59
62
  #
60
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
61
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +67,15 @@ module Sidekiq
64
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
65
68
  #
66
69
  # Example:
67
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
68
71
  #
69
72
  def push(item)
70
73
  normed = normalize_item(item)
71
- payload = process_single(item["class"], normed)
72
-
74
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
73
77
  if payload
78
+ verify_json(payload)
74
79
  raw_push([payload])
75
80
  payload["jid"]
76
81
  end
@@ -98,12 +103,17 @@ module Sidekiq
98
103
  raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
99
104
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
100
105
 
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
108
+
101
109
  normed = normalize_item(items)
102
110
  payloads = args.map.with_index { |job_args, index|
103
- copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
104
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
105
-
106
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
107
117
  result || nil
108
118
  }.compact
109
119
 
@@ -116,8 +126,8 @@ module Sidekiq
116
126
  #
117
127
  # pool = ConnectionPool.new { Redis.new }
118
128
  # Sidekiq::Client.via(pool) do
119
- # SomeWorker.perform_async(1,2,3)
120
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
121
131
  # end
122
132
  #
123
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -142,10 +152,10 @@ module Sidekiq
142
152
  end
143
153
 
144
154
  # Resque compatibility helpers. Note all helpers
145
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
146
156
  #
147
157
  # Example usage:
148
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
149
159
  #
150
160
  # Messages are enqueued to the 'default' queue.
151
161
  #
@@ -154,19 +164,19 @@ module Sidekiq
154
164
  end
155
165
 
156
166
  # Example usage:
157
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
158
168
  #
159
169
  def enqueue_to(queue, klass, *args)
160
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
161
171
  end
162
172
 
163
173
  # Example usage:
164
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
165
175
  #
166
176
  def enqueue_to_in(queue, interval, klass, *args)
167
177
  int = interval.to_f
168
178
  now = Time.now.to_f
169
- ts = (int < 1_000_000_000 ? now + int : int)
179
+ ts = ((int < 1_000_000_000) ? now + int : int)
170
180
 
171
181
  item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
172
182
  item.delete("at") if ts <= now
@@ -175,7 +185,7 @@ module Sidekiq
175
185
  end
176
186
 
177
187
  # Example usage:
178
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
179
189
  #
180
190
  def enqueue_in(interval, klass, *args)
181
191
  klass.perform_in(interval, *args)
@@ -186,8 +196,23 @@ module Sidekiq
186
196
 
187
197
  def raw_push(payloads)
188
198
  @redis_pool.with do |conn|
189
- conn.pipelined do
190
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue RedisConnection.adapter::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
191
216
  end
192
217
  end
193
218
  true
@@ -195,7 +220,7 @@ module Sidekiq
195
220
 
196
221
  def atomic_push(conn, payloads)
197
222
  if payloads.first.key?("at")
198
- conn.zadd("schedule", payloads.map { |hash|
223
+ conn.zadd("schedule", payloads.flat_map { |hash|
199
224
  at = hash.delete("at").to_s
200
225
  [at, Sidekiq.dump_json(hash)]
201
226
  })
@@ -206,54 +231,9 @@ module Sidekiq
206
231
  entry["enqueued_at"] = now
207
232
  Sidekiq.dump_json(entry)
208
233
  }
209
- conn.sadd("queues", queue)
234
+ conn.sadd("queues", [queue])
210
235
  conn.lpush("queue:#{queue}", to_push)
211
236
  end
212
237
  end
213
-
214
- def process_single(worker_class, item)
215
- queue = item["queue"]
216
-
217
- middleware.invoke(worker_class, item, queue, @redis_pool) do
218
- item
219
- end
220
- end
221
-
222
- def validate(item)
223
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
224
- raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
225
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
226
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
227
- raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
228
- end
229
-
230
- def normalize_item(item)
231
- validate(item)
232
- # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
233
-
234
- # merge in the default sidekiq_options for the item's class and/or wrapped element
235
- # this allows ActiveJobs to control sidekiq_options too.
236
- defaults = normalized_hash(item["class"])
237
- defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
238
- item = defaults.merge(item)
239
-
240
- raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
241
-
242
- item["class"] = item["class"].to_s
243
- item["queue"] = item["queue"].to_s
244
- item["jid"] ||= SecureRandom.hex(12)
245
- item["created_at"] ||= Time.now.to_f
246
-
247
- item
248
- end
249
-
250
- def normalized_hash(item_class)
251
- if item_class.is_a?(Class)
252
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
253
- item_class.get_sidekiq_options
254
- else
255
- Sidekiq.default_worker_options
256
- end
257
- end
258
238
  end
259
239
  end
@@ -1,43 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
- require "forwardable"
4
- require "socket"
5
- require "securerandom"
6
- require "sidekiq/exception_handler"
7
-
8
1
  module Sidekiq
9
2
  ##
10
- # This module is part of Sidekiq core and not intended for extensions.
11
- #
12
-
13
- class RingBuffer
14
- include Enumerable
15
- extend Forwardable
16
- def_delegators :@buf, :[], :each, :size
17
-
18
- def initialize(size, default = 0)
19
- @size = size
20
- @buf = Array.new(size, default)
21
- @index = 0
22
- end
23
-
24
- def <<(element)
25
- @buf[@index % @size] = element
26
- @index += 1
27
- element
28
- end
29
-
30
- def buffer
31
- @buf
32
- end
33
-
34
- def reset(default = 0)
35
- @buf.fill(default)
36
- end
37
- end
38
-
39
- module Util
40
- include ExceptionHandler
3
+ # Sidekiq::Component assumes a config instance is available at @config
4
+ module Component # :nodoc:
5
+ attr_reader :config
41
6
 
42
7
  def watchdog(last_words)
43
8
  yield
@@ -54,11 +19,11 @@ module Sidekiq
54
19
  end
55
20
 
56
21
  def logger
57
- Sidekiq.logger
22
+ config.logger
58
23
  end
59
24
 
60
25
  def redis(&block)
61
- Sidekiq.redis(&block)
26
+ config.redis(&block)
62
27
  end
63
28
 
64
29
  def tid
@@ -77,11 +42,16 @@ module Sidekiq
77
42
  @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
78
43
  end
79
44
 
45
+ def handle_exception(ex, ctx = {})
46
+ config.handle_exception(ex, ctx)
47
+ end
48
+
80
49
  def fire_event(event, options = {})
50
+ oneshot = options.fetch(:oneshot, true)
81
51
  reverse = options[:reverse]
82
52
  reraise = options[:reraise]
83
53
 
84
- arr = Sidekiq.options[:lifecycle_events][event]
54
+ arr = config[:lifecycle_events][event]
85
55
  arr.reverse! if reverse
86
56
  arr.each do |block|
87
57
  block.call
@@ -89,7 +59,7 @@ module Sidekiq
89
59
  handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
90
60
  raise ex if reraise
91
61
  end
92
- arr.clear
62
+ arr.clear if oneshot # once we've fired an event, we never fire it again
93
63
  end
94
64
  end
95
65
  end
data/lib/sidekiq/delay.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Sidekiq
3
+ module Sidekiq # :nodoc:
4
4
  module Extensions
5
5
  def self.enable_delay!
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
+
6
8
  if defined?(::ActiveSupport)
7
9
  require "sidekiq/extensions/active_record"
8
10
  require "sidekiq/extensions/action_mailer"
@@ -10,7 +10,7 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
16
  def method_missing(name, *args)
data/lib/sidekiq/fetch.rb CHANGED
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
+ require "sidekiq/component"
4
5
 
5
- module Sidekiq
6
+ module Sidekiq # :nodoc:
6
7
  class BasicFetch
8
+ include Sidekiq::Component
7
9
  # We want the fetch operation to timeout every few seconds so the thread
8
10
  # can check if the process is shutting down.
9
11
  TIMEOUT = 2
10
12
 
11
- UnitOfWork = Struct.new(:queue, :job) {
13
+ UnitOfWork = Struct.new(:queue, :job, :config) {
12
14
  def acknowledge
13
15
  # nothing to do
14
16
  end
@@ -18,20 +20,20 @@ module Sidekiq
18
20
  end
19
21
 
20
22
  def requeue
21
- Sidekiq.redis do |conn|
23
+ config.redis do |conn|
22
24
  conn.rpush(queue, job)
23
25
  end
24
26
  end
25
27
  }
26
28
 
27
- def initialize(options)
28
- raise ArgumentError, "missing queue list" unless options[:queues]
29
- @options = options
30
- @strictly_ordered_queues = !!@options[:strict]
31
- @queues = @options[:queues].map { |q| "queue:#{q}" }
29
+ def initialize(config)
30
+ raise ArgumentError, "missing queue list" unless config[:queues]
31
+ @config = config
32
+ @strictly_ordered_queues = !!@config[:strict]
33
+ @queues = @config[:queues].map { |q| "queue:#{q}" }
32
34
  if @strictly_ordered_queues
33
35
  @queues.uniq!
34
- @queues << TIMEOUT
36
+ @queues << {timeout: TIMEOUT}
35
37
  end
36
38
  end
37
39
 
@@ -44,30 +46,30 @@ module Sidekiq
44
46
  return nil
45
47
  end
46
48
 
47
- work = Sidekiq.redis { |conn| conn.brpop(*qs) }
48
- UnitOfWork.new(*work) if work
49
+ queue, job = redis { |conn| conn.brpop(*qs) }
50
+ UnitOfWork.new(queue, job, config) if queue
49
51
  end
50
52
 
51
53
  def bulk_requeue(inprogress, options)
52
54
  return if inprogress.empty?
53
55
 
54
- Sidekiq.logger.debug { "Re-queueing terminated jobs" }
56
+ logger.debug { "Re-queueing terminated jobs" }
55
57
  jobs_to_requeue = {}
56
58
  inprogress.each do |unit_of_work|
57
59
  jobs_to_requeue[unit_of_work.queue] ||= []
58
60
  jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
59
61
  end
60
62
 
61
- Sidekiq.redis do |conn|
62
- conn.pipelined do
63
+ redis do |conn|
64
+ conn.pipelined do |pipeline|
63
65
  jobs_to_requeue.each do |queue, jobs|
64
- conn.rpush(queue, jobs)
66
+ pipeline.rpush(queue, jobs)
65
67
  end
66
68
  end
67
69
  end
68
- Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
70
+ logger.info("Pushed #{inprogress.size} jobs back to Redis")
69
71
  rescue => ex
70
- Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
72
+ logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
71
73
  end
72
74
 
73
75
  # Creating the Redis#brpop command takes into account any
@@ -81,7 +83,7 @@ module Sidekiq
81
83
  else
82
84
  permute = @queues.shuffle
83
85
  permute.uniq!
84
- permute << TIMEOUT
86
+ permute << {timeout: TIMEOUT}
85
87
  permute
86
88
  end
87
89
  end