sidekiq 6.4.1 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +107 -5
  3. data/README.md +14 -13
  4. data/bin/sidekiq +3 -8
  5. data/bin/sidekiqload +26 -29
  6. data/lib/sidekiq/api.rb +232 -157
  7. data/lib/sidekiq/capsule.rb +110 -0
  8. data/lib/sidekiq/cli.rb +80 -86
  9. data/lib/sidekiq/client.rb +54 -42
  10. data/lib/sidekiq/component.rb +66 -0
  11. data/lib/sidekiq/config.rb +271 -0
  12. data/lib/sidekiq/deploy.rb +62 -0
  13. data/lib/sidekiq/embedded.rb +61 -0
  14. data/lib/sidekiq/fetch.rb +20 -19
  15. data/lib/sidekiq/job.rb +375 -10
  16. data/lib/sidekiq/job_logger.rb +1 -1
  17. data/lib/sidekiq/job_retry.rb +74 -53
  18. data/lib/sidekiq/job_util.rb +17 -11
  19. data/lib/sidekiq/launcher.rb +63 -69
  20. data/lib/sidekiq/logger.rb +6 -45
  21. data/lib/sidekiq/manager.rb +33 -32
  22. data/lib/sidekiq/metrics/query.rb +153 -0
  23. data/lib/sidekiq/metrics/shared.rb +95 -0
  24. data/lib/sidekiq/metrics/tracking.rb +134 -0
  25. data/lib/sidekiq/middleware/chain.rb +84 -42
  26. data/lib/sidekiq/middleware/current_attributes.rb +18 -17
  27. data/lib/sidekiq/middleware/i18n.rb +6 -4
  28. data/lib/sidekiq/middleware/modules.rb +21 -0
  29. data/lib/sidekiq/monitor.rb +1 -1
  30. data/lib/sidekiq/paginator.rb +10 -2
  31. data/lib/sidekiq/processor.rb +56 -59
  32. data/lib/sidekiq/rails.rb +10 -9
  33. data/lib/sidekiq/redis_client_adapter.rb +118 -0
  34. data/lib/sidekiq/redis_connection.rb +13 -82
  35. data/lib/sidekiq/ring_buffer.rb +29 -0
  36. data/lib/sidekiq/scheduled.rb +65 -37
  37. data/lib/sidekiq/testing/inline.rb +4 -4
  38. data/lib/sidekiq/testing.rb +41 -68
  39. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  40. data/lib/sidekiq/version.rb +2 -1
  41. data/lib/sidekiq/web/action.rb +3 -3
  42. data/lib/sidekiq/web/application.rb +22 -6
  43. data/lib/sidekiq/web/csrf_protection.rb +3 -3
  44. data/lib/sidekiq/web/helpers.rb +21 -19
  45. data/lib/sidekiq/web.rb +3 -14
  46. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  47. data/lib/sidekiq.rb +84 -207
  48. data/sidekiq.gemspec +29 -5
  49. data/web/assets/javascripts/application.js +58 -26
  50. data/web/assets/javascripts/base-charts.js +106 -0
  51. data/web/assets/javascripts/chart.min.js +13 -0
  52. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  53. data/web/assets/javascripts/dashboard-charts.js +166 -0
  54. data/web/assets/javascripts/dashboard.js +3 -240
  55. data/web/assets/javascripts/metrics.js +236 -0
  56. data/web/assets/stylesheets/application-rtl.css +2 -91
  57. data/web/assets/stylesheets/application.css +64 -297
  58. data/web/locales/ar.yml +70 -70
  59. data/web/locales/cs.yml +62 -62
  60. data/web/locales/da.yml +52 -52
  61. data/web/locales/de.yml +65 -65
  62. data/web/locales/el.yml +43 -24
  63. data/web/locales/en.yml +82 -69
  64. data/web/locales/es.yml +68 -68
  65. data/web/locales/fa.yml +65 -65
  66. data/web/locales/fr.yml +67 -67
  67. data/web/locales/he.yml +65 -64
  68. data/web/locales/hi.yml +59 -59
  69. data/web/locales/it.yml +53 -53
  70. data/web/locales/ja.yml +71 -68
  71. data/web/locales/ko.yml +52 -52
  72. data/web/locales/lt.yml +66 -66
  73. data/web/locales/nb.yml +61 -61
  74. data/web/locales/nl.yml +52 -52
  75. data/web/locales/pl.yml +45 -45
  76. data/web/locales/pt-br.yml +63 -55
  77. data/web/locales/pt.yml +51 -51
  78. data/web/locales/ru.yml +67 -66
  79. data/web/locales/sv.yml +53 -53
  80. data/web/locales/ta.yml +60 -60
  81. data/web/locales/uk.yml +62 -61
  82. data/web/locales/ur.yml +64 -64
  83. data/web/locales/vi.yml +67 -67
  84. data/web/locales/zh-cn.yml +37 -11
  85. data/web/locales/zh-tw.yml +42 -8
  86. data/web/views/_footer.erb +5 -2
  87. data/web/views/_nav.erb +1 -1
  88. data/web/views/_summary.erb +1 -1
  89. data/web/views/busy.erb +9 -4
  90. data/web/views/dashboard.erb +36 -4
  91. data/web/views/metrics.erb +80 -0
  92. data/web/views/metrics_for_job.erb +69 -0
  93. data/web/views/queue.erb +5 -1
  94. metadata +69 -22
  95. data/lib/sidekiq/delay.rb +0 -43
  96. data/lib/sidekiq/exception_handler.rb +0 -27
  97. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  98. data/lib/sidekiq/extensions/active_record.rb +0 -43
  99. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  100. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  101. data/lib/sidekiq/util.rb +0 -108
  102. data/lib/sidekiq/worker.rb +0 -362
  103. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -0,0 +1,110 @@
1
+ require "sidekiq/component"
2
+
3
+ module Sidekiq
4
+ # A Sidekiq::Capsule is the set of resources necessary to
5
+ # process one or more queues with a given concurrency.
6
+ # One "default" Capsule is started but the user may declare additional
7
+ # Capsules in their initializer.
8
+ #
9
+ # This capsule will pull jobs from the "single" queue and process
10
+ # the jobs with one thread, meaning the jobs will be processed serially.
11
+ #
12
+ # Sidekiq.configure_server do |config|
13
+ # config.capsule("single-threaded") do |cap|
14
+ # cap.concurrency = 1
15
+ # cap.queues = %w(single)
16
+ # end
17
+ # end
18
+ class Capsule
19
+ include Sidekiq::Component
20
+
21
+ attr_reader :name
22
+ attr_reader :queues
23
+ attr_accessor :concurrency
24
+
25
+ def initialize(name, config)
26
+ @name = name
27
+ @config = config
28
+ @queues = ["default"]
29
+ @concurrency = config[:concurrency]
30
+ end
31
+
32
+ def fetcher
33
+ @fetcher ||= begin
34
+ inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
35
+ inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
36
+ inst
37
+ end
38
+ end
39
+
40
+ def stop
41
+ fetcher&.bulk_requeue([])
42
+ end
43
+
44
+ def queues=(val)
45
+ @queues = Array(val).each_with_object([]) do |qstr, memo|
46
+ arr = qstr
47
+ arr = qstr.split(",") if qstr.is_a?(String)
48
+ name, weight = arr
49
+ [weight.to_i, 1].max.times do
50
+ memo << name
51
+ end
52
+ end
53
+ end
54
+
55
+ # Allow the middleware to be different per-capsule.
56
+ # Avoid if possible and add middleware globally so all
57
+ # capsules share the same chains. Easier to debug that way.
58
+ def client_middleware
59
+ @client_chain ||= config.client_middleware.copy_for(self)
60
+ yield @client_chain if block_given?
61
+ @client_chain
62
+ end
63
+
64
+ def server_middleware
65
+ @server_chain ||= config.server_middleware.copy_for(self)
66
+ yield @server_chain if block_given?
67
+ @server_chain
68
+ end
69
+
70
+ def redis_pool
71
+ Thread.current[:sidekiq_redis_pool] || local_redis_pool
72
+ end
73
+
74
+ def local_redis_pool
75
+ # connection pool is lazy, it will not create connections unless you actually need them
76
+ # so don't be skimpy!
77
+ @redis ||= config.new_redis_pool(@concurrency, name)
78
+ end
79
+
80
+ def redis
81
+ raise ArgumentError, "requires a block" unless block_given?
82
+ redis_pool.with do |conn|
83
+ retryable = true
84
+ begin
85
+ yield conn
86
+ rescue RedisClientAdapter::BaseError => ex
87
+ # 2550 Failover can cause the server to become a replica, need
88
+ # to disconnect and reopen the socket to get back to the primary.
89
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
90
+ # 4985 Use the same logic when a blocking command is force-unblocked
91
+ # The same retry logic is also used in client.rb
92
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
93
+ conn.close
94
+ retryable = false
95
+ retry
96
+ end
97
+ raise
98
+ end
99
+ end
100
+ end
101
+
102
+ def lookup(name)
103
+ config.lookup(name)
104
+ end
105
+
106
+ def logger
107
+ config.logger
108
+ end
109
+ end
110
+ end
data/lib/sidekiq/cli.rb CHANGED
@@ -9,18 +9,23 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/config"
13
+ require "sidekiq/component"
14
+ require "sidekiq/capsule"
12
15
  require "sidekiq/launcher"
13
- require "sidekiq/util"
14
16
 
15
- module Sidekiq
17
+ module Sidekiq # :nodoc:
16
18
  class CLI
17
- include Util
19
+ include Sidekiq::Component
18
20
  include Singleton unless $TESTING
19
21
 
20
22
  attr_accessor :launcher
21
23
  attr_accessor :environment
24
+ attr_accessor :config
25
+
26
+ def parse(args = ARGV.dup)
27
+ @config ||= Sidekiq.default_configuration
22
28
 
23
- def parse(args = ARGV)
24
29
  setup_options(args)
25
30
  initialize_logger
26
31
  validate!
@@ -36,7 +41,7 @@ module Sidekiq
36
41
  def run(boot_app: true)
37
42
  boot_application if boot_app
38
43
 
39
- if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
40
45
  print_banner
41
46
  end
42
47
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,9 +72,9 @@ module Sidekiq
67
72
 
68
73
  # touch the connection pool so it is created before we
69
74
  # fire startup and start multithreading.
70
- info = Sidekiq.redis_info
71
- ver = info["redis_version"]
72
- raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
75
+ info = @config.redis_info
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
73
78
 
74
79
  maxmemory_policy = info["maxmemory_policy"]
75
80
  if maxmemory_policy != "noeviction"
@@ -85,22 +90,22 @@ module Sidekiq
85
90
 
86
91
  # Since the user can pass us a connection pool explicitly in the initializer, we
87
92
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
88
- cursize = Sidekiq.redis_pool.size
89
- needed = Sidekiq.options[:concurrency] + 2
90
- raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
93
+ @config.capsules.each_pair do |name, cap|
94
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
95
+ end
91
96
 
92
97
  # cache process identity
93
- Sidekiq.options[:identity] = identity
98
+ @config[:identity] = identity
94
99
 
95
100
  # Touch middleware so it isn't lazy loaded by multiple threads, #3043
96
- Sidekiq.server_middleware
101
+ @config.server_middleware
97
102
 
98
103
  # Before this point, the process is initializing with just the main thread.
99
104
  # Starting here the process will now have multiple threads running.
100
105
  fire_event(:startup, reverse: false, reraise: true)
101
106
 
102
- logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
103
- logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
107
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
108
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
104
109
 
105
110
  launch(self_read)
106
111
  end
@@ -110,13 +115,13 @@ module Sidekiq
110
115
  logger.info "Starting processing, hit Ctrl-C to stop"
111
116
  end
112
117
 
113
- @launcher = Sidekiq::Launcher.new(options)
118
+ @launcher = Sidekiq::Launcher.new(@config)
114
119
 
115
120
  begin
116
121
  launcher.run
117
122
 
118
- while (readable_io = self_read.wait_readable)
119
- signal = readable_io.gets.strip
123
+ while self_read.wait_readable
124
+ signal = self_read.gets.strip
120
125
  handle_signal(signal)
121
126
  end
122
127
  rescue Interrupt
@@ -133,19 +138,34 @@ module Sidekiq
133
138
  end
134
139
  end
135
140
 
136
- def self.w
137
- "\e[37m"
141
+ HOLIDAY_COLORS = {
142
+ # got other color-specific holidays from around the world?
143
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
144
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
145
+ "10-31" => "\e[38;5;208m" # Halloween orange
146
+ }
147
+
148
+ def self.day
149
+ @@day ||= begin
150
+ t = Date.today
151
+ "#{t.month}-#{t.day}"
152
+ end
138
153
  end
139
154
 
140
155
  def self.r
141
- "\e[31m"
156
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
142
157
  end
143
158
 
144
159
  def self.b
145
- "\e[30m"
160
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
161
+ end
162
+
163
+ def self.w
164
+ "\e[1;37m"
146
165
  end
147
166
 
148
167
  def self.reset
168
+ @@b = @@r = @@day = nil
149
169
  "\e[0m"
150
170
  end
151
171
 
@@ -158,7 +178,7 @@ module Sidekiq
158
178
  #{w} ,$$$$$b#{b}/#{w}md$$$P^'
159
179
  #{w} .d$$$$$$#{b}/#{w}$$$P'
160
180
  #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
161
- #{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
181
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
162
182
  #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
163
183
  #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
164
184
  #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
@@ -173,25 +193,25 @@ module Sidekiq
173
193
  # Heroku sends TERM and then waits 30 seconds for process to exit.
174
194
  "TERM" => ->(cli) { raise Interrupt },
175
195
  "TSTP" => ->(cli) {
176
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
196
+ cli.logger.info "Received TSTP, no longer accepting new work"
177
197
  cli.launcher.quiet
178
198
  },
179
199
  "TTIN" => ->(cli) {
180
200
  Thread.list.each do |thread|
181
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
201
+ cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
182
202
  if thread.backtrace
183
- Sidekiq.logger.warn thread.backtrace.join("\n")
203
+ cli.logger.warn thread.backtrace.join("\n")
184
204
  else
185
- Sidekiq.logger.warn "<no backtrace available>"
205
+ cli.logger.warn "<no backtrace available>"
186
206
  end
187
207
  end
188
208
  }
189
209
  }
190
- UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
210
+ UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
191
211
  SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
192
212
 
193
213
  def handle_signal(sig)
194
- Sidekiq.logger.debug "Got #{sig} signal"
214
+ logger.debug "Got #{sig} signal"
195
215
  SIGNAL_HANDLERS[sig].call(self)
196
216
  end
197
217
 
@@ -237,7 +257,7 @@ module Sidekiq
237
257
  config_dir = if File.directory?(opts[:require].to_s)
238
258
  File.join(opts[:require], "config")
239
259
  else
240
- File.join(options[:require], "config")
260
+ File.join(@config[:require], "config")
241
261
  end
242
262
 
243
263
  %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +274,34 @@ module Sidekiq
254
274
  opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
255
275
 
256
276
  # merge with defaults
257
- options.merge!(opts)
258
- end
277
+ @config.merge!(opts)
278
+
279
+ @config.default_capsule.tap do |cap|
280
+ cap.queues = opts[:queues]
281
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
282
+ end
259
283
 
260
- def options
261
- Sidekiq.options
284
+ opts[:capsules]&.each do |name, cap_config|
285
+ @config.capsule(name.to_s) do |cap|
286
+ cap.queues = cap_config[:queues]
287
+ cap.concurrency = cap_config[:concurrency]
288
+ end
289
+ end
262
290
  end
263
291
 
264
292
  def boot_application
265
293
  ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
266
294
 
267
- if File.directory?(options[:require])
295
+ if File.directory?(@config[:require])
268
296
  require "rails"
269
- if ::Rails::VERSION::MAJOR < 5
270
- raise "Sidekiq no longer supports this version of Rails"
271
- else
272
- require "sidekiq/rails"
273
- require File.expand_path("#{options[:require]}/config/environment.rb")
297
+ if ::Rails::VERSION::MAJOR < 6
298
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
274
299
  end
275
- options[:tag] ||= default_tag
300
+ require "sidekiq/rails"
301
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
302
+ @config[:tag] ||= default_tag
276
303
  else
277
- require options[:require]
304
+ require @config[:require]
278
305
  end
279
306
  end
280
307
 
@@ -291,18 +318,18 @@ module Sidekiq
291
318
  end
292
319
 
293
320
  def validate!
294
- if !File.exist?(options[:require]) ||
295
- (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
321
+ if !File.exist?(@config[:require]) ||
322
+ (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
296
323
  logger.info "=================================================================="
297
324
  logger.info " Please point Sidekiq to a Rails application or a Ruby file "
298
- logger.info " to load your worker classes with -r [DIR|FILE]."
325
+ logger.info " to load your job classes with -r [DIR|FILE]."
299
326
  logger.info "=================================================================="
300
327
  logger.info @parser
301
328
  die(1)
302
329
  end
303
330
 
304
331
  [:concurrency, :timeout].each do |opt|
305
- raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
332
+ raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
306
333
  end
307
334
  end
308
335
 
@@ -319,10 +346,6 @@ module Sidekiq
319
346
  opts[:concurrency] = Integer(arg)
320
347
  end
321
348
 
322
- o.on "-d", "--daemon", "Daemonize process" do |arg|
323
- puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
324
- end
325
-
326
349
  o.on "-e", "--environment ENV", "Application environment" do |arg|
327
350
  opts[:environment] = arg
328
351
  end
@@ -332,11 +355,11 @@ module Sidekiq
332
355
  end
333
356
 
334
357
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
335
- queue, weight = arg.split(",")
336
- parse_queue opts, queue, weight
358
+ opts[:queues] ||= []
359
+ opts[:queues] << arg
337
360
  end
338
361
 
339
- o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
362
+ o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
340
363
  opts[:require] = arg
341
364
  end
342
365
 
@@ -352,15 +375,7 @@ module Sidekiq
352
375
  opts[:config_file] = arg
353
376
  end
354
377
 
355
- o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
356
- puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
357
- end
358
-
359
- o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
360
- puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
361
- end
362
-
363
- o.on "-V", "--version", "Print version and exit" do |arg|
378
+ o.on "-V", "--version", "Print version and exit" do
364
379
  puts "Sidekiq #{Sidekiq::VERSION}"
365
380
  die(0)
366
381
  end
@@ -376,13 +391,13 @@ module Sidekiq
376
391
  end
377
392
 
378
393
  def initialize_logger
379
- Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
394
+ @config.logger.level = ::Logger::DEBUG if @config[:verbose]
380
395
  end
381
396
 
382
397
  def parse_config(path)
383
398
  erb = ERB.new(File.read(path))
384
399
  erb.filename = File.expand_path(path)
385
- opts = load_yaml(erb.result) || {}
400
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
386
401
 
387
402
  if opts.respond_to? :deep_symbolize_keys!
388
403
  opts.deep_symbolize_keys!
@@ -393,31 +408,9 @@ module Sidekiq
393
408
  opts = opts.merge(opts.delete(environment.to_sym) || {})
394
409
  opts.delete(:strict)
395
410
 
396
- parse_queues(opts, opts.delete(:queues) || [])
397
-
398
411
  opts
399
412
  end
400
413
 
401
- def load_yaml(src)
402
- if Psych::VERSION > "4.0"
403
- YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
404
- else
405
- YAML.load(src)
406
- end
407
- end
408
-
409
- def parse_queues(opts, queues_and_weights)
410
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
411
- end
412
-
413
- def parse_queue(opts, queue, weight = nil)
414
- opts[:queues] ||= []
415
- opts[:strict] = true if opts[:strict].nil?
416
- raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
417
- [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
418
- opts[:strict] = false if weight.to_i > 0
419
- end
420
-
421
414
  def rails_app?
422
415
  defined?(::Rails) && ::Rails.respond_to?(:application)
423
416
  end
@@ -425,3 +418,4 @@ module Sidekiq
425
418
  end
426
419
 
427
420
  require "sidekiq/systemd"
421
+ require "sidekiq/metrics/tracking"
@@ -15,13 +15,12 @@ module Sidekiq
15
15
  # client.middleware do |chain|
16
16
  # chain.use MyClientMiddleware
17
17
  # end
18
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
19
19
  #
20
20
  # All client instances default to the globally-defined
21
21
  # Sidekiq.client_middleware but you can change as necessary.
22
22
  #
23
23
  def middleware(&block)
24
- @chain ||= Sidekiq.client_middleware
25
24
  if block
26
25
  @chain = @chain.dup
27
26
  yield @chain
@@ -31,34 +30,48 @@ module Sidekiq
31
30
 
32
31
  attr_accessor :redis_pool
33
32
 
34
- # Sidekiq::Client normally uses the default Redis pool but you may
35
- # pass a custom ConnectionPool if you want to shard your
36
- # Sidekiq jobs across several Redis instances (for scalability
37
- # reasons, e.g.)
33
+ # Sidekiq::Client is responsible for pushing job payloads to Redis.
34
+ # Requires the :pool or :config keyword argument.
38
35
  #
39
- # Sidekiq::Client.new(ConnectionPool.new { Redis.new })
36
+ # Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
40
37
  #
41
- # Generally this is only needed for very large Sidekiq installs processing
42
- # thousands of jobs per second. I don't recommend sharding unless you
43
- # cannot scale any other way (e.g. splitting your app into smaller apps).
44
- def initialize(redis_pool = nil)
45
- @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
38
+ # Inside the Sidekiq process, you can reuse the configured resources:
39
+ #
40
+ # Sidekiq::Client.new(config: config)
41
+ #
42
+ # @param pool [ConnectionPool] explicit Redis pool to use
43
+ # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
44
+ # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
45
+ def initialize(*args, **kwargs)
46
+ if args.size == 1 && kwargs.size == 0
47
+ warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
48
+ # old calling method, accept 1 pool argument
49
+ @redis_pool = args[0]
50
+ @chain = Sidekiq.default_configuration.client_middleware
51
+ @config = Sidekiq.default_configuration
52
+ else
53
+ # new calling method: keyword arguments
54
+ @config = kwargs[:config] || Sidekiq.default_configuration
55
+ @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
56
+ @chain = kwargs[:chain] || @config&.client_middleware
57
+ raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
58
+ end
46
59
  end
47
60
 
48
61
  ##
49
62
  # The main method used to push a job to Redis. Accepts a number of options:
50
63
  #
51
64
  # queue - the named queue to use, default 'default'
52
- # class - the worker class to call, required
65
+ # class - the job class to call, required
53
66
  # args - an array of simple arguments to the perform method, must be JSON-serializable
54
67
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
55
68
  # retry - whether to retry this job if it fails, default true or an integer number of retries
56
69
  # backtrace - whether to save any error backtrace, default false
57
70
  #
58
71
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
59
- # worker options. Otherwise, they will be based on the job class's options.
72
+ # job options. Otherwise, they will be based on the job class's options.
60
73
  #
61
- # Any options valid for a worker class's sidekiq_options are also available here.
74
+ # Any options valid for a job class's sidekiq_options are also available here.
62
75
  #
63
76
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
64
77
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -67,13 +80,15 @@ module Sidekiq
67
80
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
68
81
  #
69
82
  # Example:
70
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
83
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
71
84
  #
72
85
  def push(item)
73
86
  normed = normalize_item(item)
74
- payload = process_single(item["class"], normed)
75
-
87
+ payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
88
+ normed
89
+ end
76
90
  if payload
91
+ verify_json(payload)
77
92
  raw_push([payload])
78
93
  payload["jid"]
79
94
  end
@@ -101,12 +116,17 @@ module Sidekiq
101
116
  raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
102
117
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
103
118
 
119
+ jid = items.delete("jid")
120
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
121
+
104
122
  normed = normalize_item(items)
105
123
  payloads = args.map.with_index { |job_args, index|
106
124
  copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
107
125
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
108
-
109
- result = process_single(items["class"], copy)
126
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
127
+ verify_json(copy)
128
+ copy
129
+ end
110
130
  result || nil
111
131
  }.compact
112
132
 
@@ -119,8 +139,8 @@ module Sidekiq
119
139
  #
120
140
  # pool = ConnectionPool.new { Redis.new }
121
141
  # Sidekiq::Client.via(pool) do
122
- # SomeWorker.perform_async(1,2,3)
123
- # SomeOtherWorker.perform_async(1,2,3)
142
+ # SomeJob.perform_async(1,2,3)
143
+ # SomeOtherJob.perform_async(1,2,3)
124
144
  # end
125
145
  #
126
146
  # Generally this is only needed for very large Sidekiq installs processing
@@ -128,11 +148,11 @@ module Sidekiq
128
148
  # you cannot scale any other way (e.g. splitting your app into smaller apps).
129
149
  def self.via(pool)
130
150
  raise ArgumentError, "No pool given" if pool.nil?
131
- current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
132
- Thread.current[:sidekiq_via_pool] = pool
151
+ current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
152
+ Thread.current[:sidekiq_redis_pool] = pool
133
153
  yield
134
154
  ensure
135
- Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
155
+ Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
136
156
  end
137
157
 
138
158
  class << self
@@ -145,10 +165,10 @@ module Sidekiq
145
165
  end
146
166
 
147
167
  # Resque compatibility helpers. Note all helpers
148
- # should go through Worker#client_push.
168
+ # should go through Sidekiq::Job#client_push.
149
169
  #
150
170
  # Example usage:
151
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
171
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
152
172
  #
153
173
  # Messages are enqueued to the 'default' queue.
154
174
  #
@@ -157,14 +177,14 @@ module Sidekiq
157
177
  end
158
178
 
159
179
  # Example usage:
160
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
180
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
161
181
  #
162
182
  def enqueue_to(queue, klass, *args)
163
183
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
164
184
  end
165
185
 
166
186
  # Example usage:
167
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
187
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
168
188
  #
169
189
  def enqueue_to_in(queue, interval, klass, *args)
170
190
  int = interval.to_f
@@ -178,7 +198,7 @@ module Sidekiq
178
198
  end
179
199
 
180
200
  # Example usage:
181
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
201
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
182
202
  #
183
203
  def enqueue_in(interval, klass, *args)
184
204
  klass.perform_in(interval, *args)
@@ -194,14 +214,14 @@ module Sidekiq
194
214
  conn.pipelined do |pipeline|
195
215
  atomic_push(pipeline, payloads)
196
216
  end
197
- rescue Redis::BaseError => ex
217
+ rescue RedisClient::Error => ex
198
218
  # 2550 Failover can cause the server to become a replica, need
199
219
  # to disconnect and reopen the socket to get back to the primary.
200
220
  # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
201
221
  # 4985 Use the same logic when a blocking command is force-unblocked
202
222
  # The retry logic is copied from sidekiq.rb
203
223
  if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
204
- conn.disconnect!
224
+ conn.close
205
225
  retryable = false
206
226
  retry
207
227
  end
@@ -213,7 +233,7 @@ module Sidekiq
213
233
 
214
234
  def atomic_push(conn, payloads)
215
235
  if payloads.first.key?("at")
216
- conn.zadd("schedule", payloads.map { |hash|
236
+ conn.zadd("schedule", payloads.flat_map { |hash|
217
237
  at = hash.delete("at").to_s
218
238
  [at, Sidekiq.dump_json(hash)]
219
239
  })
@@ -224,17 +244,9 @@ module Sidekiq
224
244
  entry["enqueued_at"] = now
225
245
  Sidekiq.dump_json(entry)
226
246
  }
227
- conn.sadd("queues", queue)
247
+ conn.sadd("queues", [queue])
228
248
  conn.lpush("queue:#{queue}", to_push)
229
249
  end
230
250
  end
231
-
232
- def process_single(worker_class, item)
233
- queue = item["queue"]
234
-
235
- middleware.invoke(worker_class, item, queue, @redis_pool) do
236
- item
237
- end
238
- end
239
251
  end
240
252
  end