sidekiq 6.5.9 → 7.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +5 -13
  3. data/README.md +13 -12
  4. data/bin/sidekiq +3 -8
  5. data/bin/sidekiqload +14 -23
  6. data/lib/sidekiq/api.rb +51 -127
  7. data/lib/sidekiq/capsule.rb +110 -0
  8. data/lib/sidekiq/cli.rb +44 -59
  9. data/lib/sidekiq/client.rb +30 -17
  10. data/lib/sidekiq/component.rb +1 -0
  11. data/lib/sidekiq/config.rb +270 -0
  12. data/lib/sidekiq/deploy.rb +62 -0
  13. data/lib/sidekiq/embedded.rb +61 -0
  14. data/lib/sidekiq/fetch.rb +10 -11
  15. data/lib/sidekiq/job.rb +375 -10
  16. data/lib/sidekiq/job_logger.rb +1 -1
  17. data/lib/sidekiq/job_retry.rb +8 -8
  18. data/lib/sidekiq/job_util.rb +4 -4
  19. data/lib/sidekiq/launcher.rb +36 -46
  20. data/lib/sidekiq/logger.rb +1 -26
  21. data/lib/sidekiq/manager.rb +9 -11
  22. data/lib/sidekiq/metrics/query.rb +3 -3
  23. data/lib/sidekiq/metrics/shared.rb +4 -3
  24. data/lib/sidekiq/metrics/tracking.rb +18 -18
  25. data/lib/sidekiq/middleware/chain.rb +7 -9
  26. data/lib/sidekiq/middleware/current_attributes.rb +3 -8
  27. data/lib/sidekiq/monitor.rb +1 -1
  28. data/lib/sidekiq/paginator.rb +1 -9
  29. data/lib/sidekiq/pool.rb +7 -0
  30. data/lib/sidekiq/processor.rb +17 -26
  31. data/lib/sidekiq/redis_client_adapter.rb +9 -45
  32. data/lib/sidekiq/redis_connection.rb +11 -111
  33. data/lib/sidekiq/scheduled.rb +19 -20
  34. data/lib/sidekiq/testing.rb +4 -32
  35. data/lib/sidekiq/transaction_aware_client.rb +4 -5
  36. data/lib/sidekiq/version.rb +1 -1
  37. data/lib/sidekiq/web/application.rb +1 -4
  38. data/lib/sidekiq/web/csrf_protection.rb +1 -1
  39. data/lib/sidekiq/web/helpers.rb +21 -22
  40. data/lib/sidekiq/web.rb +2 -17
  41. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  42. data/lib/sidekiq.rb +52 -274
  43. data/sidekiq.gemspec +29 -5
  44. data/web/assets/javascripts/base-charts.js +106 -0
  45. data/web/assets/javascripts/dashboard-charts.js +166 -0
  46. data/web/assets/javascripts/dashboard.js +3 -223
  47. data/web/assets/javascripts/metrics.js +90 -116
  48. data/web/assets/stylesheets/application-rtl.css +2 -91
  49. data/web/assets/stylesheets/application.css +21 -296
  50. data/web/locales/ar.yml +70 -70
  51. data/web/locales/cs.yml +62 -62
  52. data/web/locales/da.yml +52 -52
  53. data/web/locales/de.yml +65 -65
  54. data/web/locales/el.yml +2 -7
  55. data/web/locales/en.yml +76 -70
  56. data/web/locales/es.yml +68 -68
  57. data/web/locales/fa.yml +65 -65
  58. data/web/locales/fr.yml +67 -67
  59. data/web/locales/he.yml +65 -64
  60. data/web/locales/hi.yml +59 -59
  61. data/web/locales/it.yml +53 -53
  62. data/web/locales/ja.yml +64 -68
  63. data/web/locales/ko.yml +52 -52
  64. data/web/locales/lt.yml +66 -66
  65. data/web/locales/nb.yml +61 -61
  66. data/web/locales/nl.yml +52 -52
  67. data/web/locales/pl.yml +45 -45
  68. data/web/locales/pt-br.yml +59 -69
  69. data/web/locales/pt.yml +51 -51
  70. data/web/locales/ru.yml +67 -66
  71. data/web/locales/sv.yml +53 -53
  72. data/web/locales/ta.yml +60 -60
  73. data/web/locales/uk.yml +62 -61
  74. data/web/locales/ur.yml +64 -64
  75. data/web/locales/vi.yml +67 -67
  76. data/web/locales/zh-cn.yml +1 -0
  77. data/web/locales/zh-tw.yml +10 -1
  78. data/web/views/_footer.erb +5 -2
  79. data/web/views/busy.erb +1 -6
  80. data/web/views/dashboard.erb +36 -5
  81. data/web/views/metrics.erb +30 -19
  82. data/web/views/metrics_for_job.erb +16 -34
  83. metadata +60 -37
  84. data/lib/sidekiq/delay.rb +0 -43
  85. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  86. data/lib/sidekiq/extensions/active_record.rb +0 -43
  87. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  88. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  89. data/lib/sidekiq/metrics/deploy.rb +0 -47
  90. data/lib/sidekiq/worker.rb +0 -370
  91. data/web/assets/javascripts/graph.js +0 -16
  92. /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/cli.rb CHANGED
@@ -9,20 +9,11 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/config"
12
13
  require "sidekiq/component"
14
+ require "sidekiq/capsule"
13
15
  require "sidekiq/launcher"
14
16
 
15
- # module ScoutApm
16
- # VERSION = "5.3.1"
17
- # end
18
- fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
19
-
20
-
21
- scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
22
-
23
-
24
- EOM
25
-
26
17
  module Sidekiq # :nodoc:
27
18
  class CLI
28
19
  include Sidekiq::Component
@@ -33,9 +24,7 @@ module Sidekiq # :nodoc:
33
24
  attr_accessor :config
34
25
 
35
26
  def parse(args = ARGV.dup)
36
- @config = Sidekiq
37
- @config[:error_handlers].clear
38
- @config[:error_handlers] << @config.method(:default_error_handler)
27
+ @config ||= Sidekiq.default_configuration
39
28
 
40
29
  setup_options(args)
41
30
  initialize_logger
@@ -52,7 +41,7 @@ module Sidekiq # :nodoc:
52
41
  def run(boot_app: true)
53
42
  boot_application if boot_app
54
43
 
55
- if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
56
45
  print_banner
57
46
  end
58
47
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -84,8 +73,8 @@ module Sidekiq # :nodoc:
84
73
  # touch the connection pool so it is created before we
85
74
  # fire startup and start multithreading.
86
75
  info = @config.redis_info
87
- ver = info["redis_version"]
88
- raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
89
78
 
90
79
  maxmemory_policy = info["maxmemory_policy"]
91
80
  if maxmemory_policy != "noeviction"
@@ -101,9 +90,9 @@ module Sidekiq # :nodoc:
101
90
 
102
91
  # Since the user can pass us a connection pool explicitly in the initializer, we
103
92
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
104
- cursize = @config.redis_pool.size
105
- needed = @config[:concurrency] + 2
106
- raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
93
+ @config.capsules.each_pair do |name, cap|
94
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
95
+ end
107
96
 
108
97
  # cache process identity
109
98
  @config[:identity] = identity
@@ -115,8 +104,8 @@ module Sidekiq # :nodoc:
115
104
  # Starting here the process will now have multiple threads running.
116
105
  fire_event(:startup, reverse: false, reraise: true)
117
106
 
118
- logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
119
- logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
107
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
108
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
120
109
 
121
110
  launch(self_read)
122
111
  end
@@ -149,19 +138,34 @@ module Sidekiq # :nodoc:
149
138
  end
150
139
  end
151
140
 
152
- def self.w
153
- "\e[37m"
141
+ HOLIDAY_COLORS = {
142
+ # got other color-specific holidays from around the world?
143
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
144
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
145
+ "10-31" => "\e[38;5;208m" # Halloween orange
146
+ }
147
+
148
+ def self.day
149
+ @@day ||= begin
150
+ t = Date.today
151
+ "#{t.month}-#{t.day}"
152
+ end
154
153
  end
155
154
 
156
155
  def self.r
157
- "\e[31m"
156
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
158
157
  end
159
158
 
160
159
  def self.b
161
- "\e[30m"
160
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
161
+ end
162
+
163
+ def self.w
164
+ "\e[1;37m"
162
165
  end
163
166
 
164
167
  def self.reset
168
+ @@b = @@r = @@day = nil
165
169
  "\e[0m"
166
170
  end
167
171
 
@@ -174,7 +178,7 @@ module Sidekiq # :nodoc:
174
178
  #{w} ,$$$$$b#{b}/#{w}md$$$P^'
175
179
  #{w} .d$$$$$$#{b}/#{w}$$$P'
176
180
  #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
177
- #{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
181
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
178
182
  #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
179
183
  #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
180
184
  #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
@@ -225,7 +229,6 @@ module Sidekiq # :nodoc:
225
229
  # Both Sinatra 2.0+ and Sidekiq support this term.
226
230
  # RAILS_ENV and RACK_ENV are there for legacy support.
227
231
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
228
- config[:environment] = @environment
229
232
  end
230
233
 
231
234
  def symbolize_keys_deep!(hash)
@@ -272,6 +275,11 @@ module Sidekiq # :nodoc:
272
275
 
273
276
  # merge with defaults
274
277
  @config.merge!(opts)
278
+
279
+ @config.default_capsule.tap do |cap|
280
+ cap.queues = opts[:queues]
281
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
282
+ end
275
283
  end
276
284
 
277
285
  def boot_application
@@ -279,12 +287,11 @@ module Sidekiq # :nodoc:
279
287
 
280
288
  if File.directory?(@config[:require])
281
289
  require "rails"
282
- if ::Rails::VERSION::MAJOR < 5
283
- raise "Sidekiq no longer supports this version of Rails"
284
- else
285
- require "sidekiq/rails"
286
- require File.expand_path("#{@config[:require]}/config/environment.rb")
290
+ if ::Rails::VERSION::MAJOR < 6
291
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
287
292
  end
293
+ require "sidekiq/rails"
294
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
288
295
  @config[:tag] ||= default_tag
289
296
  else
290
297
  require @config[:require]
@@ -345,8 +352,8 @@ module Sidekiq # :nodoc:
345
352
  end
346
353
 
347
354
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
348
- queue, weight = arg.split(",")
349
- parse_queue opts, queue, weight
355
+ opts[:queues] ||= []
356
+ opts[:queues] << arg
350
357
  end
351
358
 
352
359
  o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
@@ -395,7 +402,7 @@ module Sidekiq # :nodoc:
395
402
  def parse_config(path)
396
403
  erb = ERB.new(File.read(path))
397
404
  erb.filename = File.expand_path(path)
398
- opts = load_yaml(erb.result) || {}
405
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
399
406
 
400
407
  if opts.respond_to? :deep_symbolize_keys!
401
408
  opts.deep_symbolize_keys!
@@ -406,31 +413,9 @@ module Sidekiq # :nodoc:
406
413
  opts = opts.merge(opts.delete(environment.to_sym) || {})
407
414
  opts.delete(:strict)
408
415
 
409
- parse_queues(opts, opts.delete(:queues) || [])
410
-
411
416
  opts
412
417
  end
413
418
 
414
- def load_yaml(src)
415
- if Psych::VERSION > "4.0"
416
- YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
417
- else
418
- YAML.load(src)
419
- end
420
- end
421
-
422
- def parse_queues(opts, queues_and_weights)
423
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
424
- end
425
-
426
- def parse_queue(opts, queue, weight = nil)
427
- opts[:queues] ||= []
428
- opts[:strict] = true if opts[:strict].nil?
429
- raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
430
- [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
431
- opts[:strict] = false if weight.to_i > 0
432
- end
433
-
434
419
  def rails_app?
435
420
  defined?(::Rails) && ::Rails.respond_to?(:application)
436
421
  end
@@ -438,4 +423,4 @@ module Sidekiq # :nodoc:
438
423
  end
439
424
 
440
425
  require "sidekiq/systemd"
441
- require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
426
+ require "sidekiq/metrics/tracking"
@@ -21,7 +21,6 @@ module Sidekiq
21
21
  # Sidekiq.client_middleware but you can change as necessary.
22
22
  #
23
23
  def middleware(&block)
24
- @chain ||= Sidekiq.client_middleware
25
24
  if block
26
25
  @chain = @chain.dup
27
26
  yield @chain
@@ -31,18 +30,32 @@ module Sidekiq
31
30
 
32
31
  attr_accessor :redis_pool
33
32
 
34
- # Sidekiq::Client normally uses the default Redis pool but you may
35
- # pass a custom ConnectionPool if you want to shard your
36
- # Sidekiq jobs across several Redis instances (for scalability
37
- # reasons, e.g.)
33
+ # Sidekiq::Client is responsible for pushing job payloads to Redis.
34
+ # Requires the :pool or :config keyword argument.
38
35
  #
39
- # Sidekiq::Client.new(ConnectionPool.new { Redis.new })
36
+ # Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
40
37
  #
41
- # Generally this is only needed for very large Sidekiq installs processing
42
- # thousands of jobs per second. I don't recommend sharding unless you
43
- # cannot scale any other way (e.g. splitting your app into smaller apps).
44
- def initialize(redis_pool = nil)
45
- @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
38
+ # Inside the Sidekiq process, you can reuse the configured resources:
39
+ #
40
+ # Sidekiq::Client.new(config: config)
41
+ #
42
+ # @param pool [ConnectionPool] explicit Redis pool to use
43
+ # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
44
+ # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
45
+ def initialize(*args, **kwargs)
46
+ if args.size == 1 && kwargs.size == 0
47
+ warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
48
+ # old calling method, accept 1 pool argument
49
+ @redis_pool = args[0]
50
+ @chain = Sidekiq.default_configuration.client_middleware
51
+ @config = Sidekiq.default_configuration
52
+ else
53
+ # new calling method: keyword arguments
54
+ @config = kwargs[:config] || Sidekiq.default_configuration
55
+ @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
56
+ @chain = kwargs[:chain] || @config&.client_middleware
57
+ raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
58
+ end
46
59
  end
47
60
 
48
61
  ##
@@ -135,11 +148,11 @@ module Sidekiq
135
148
  # you cannot scale any other way (e.g. splitting your app into smaller apps).
136
149
  def self.via(pool)
137
150
  raise ArgumentError, "No pool given" if pool.nil?
138
- current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
139
- Thread.current[:sidekiq_via_pool] = pool
151
+ current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
152
+ Thread.current[:sidekiq_redis_pool] = pool
140
153
  yield
141
154
  ensure
142
- Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
155
+ Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
143
156
  end
144
157
 
145
158
  class << self
@@ -176,7 +189,7 @@ module Sidekiq
176
189
  def enqueue_to_in(queue, interval, klass, *args)
177
190
  int = interval.to_f
178
191
  now = Time.now.to_f
179
- ts = ((int < 1_000_000_000) ? now + int : int)
192
+ ts = (int < 1_000_000_000 ? now + int : int)
180
193
 
181
194
  item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
182
195
  item.delete("at") if ts <= now
@@ -201,14 +214,14 @@ module Sidekiq
201
214
  conn.pipelined do |pipeline|
202
215
  atomic_push(pipeline, payloads)
203
216
  end
204
- rescue RedisConnection.adapter::BaseError => ex
217
+ rescue RedisClient::Error => ex
205
218
  # 2550 Failover can cause the server to become a replica, need
206
219
  # to disconnect and reopen the socket to get back to the primary.
207
220
  # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
221
  # 4985 Use the same logic when a blocking command is force-unblocked
209
222
  # The retry logic is copied from sidekiq.rb
210
223
  if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
- conn.disconnect!
224
+ conn.close
212
225
  retryable = false
213
226
  retry
214
227
  end
@@ -50,6 +50,7 @@ module Sidekiq
50
50
  oneshot = options.fetch(:oneshot, true)
51
51
  reverse = options[:reverse]
52
52
  reraise = options[:reraise]
53
+ logger.debug("Firing #{event} event") if oneshot
53
54
 
54
55
  arr = config[:lifecycle_events][event]
55
56
  arr.reverse! if reverse
@@ -0,0 +1,270 @@
1
+ require "forwardable"
2
+
3
+ require "set"
4
+ require "sidekiq/redis_connection"
5
+
6
+ module Sidekiq
7
+ # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
8
+ class Config
9
+ extend Forwardable
10
+
11
+ DEFAULTS = {
12
+ labels: Set.new,
13
+ require: ".",
14
+ environment: nil,
15
+ concurrency: 5,
16
+ timeout: 25,
17
+ poll_interval_average: nil,
18
+ average_scheduled_poll_interval: 5,
19
+ on_complex_arguments: :raise,
20
+ error_handlers: [],
21
+ death_handlers: [],
22
+ lifecycle_events: {
23
+ startup: [],
24
+ quiet: [],
25
+ shutdown: [],
26
+ # triggers when we fire the first heartbeat on startup OR repairing a network partition
27
+ heartbeat: [],
28
+ # triggers on EVERY heartbeat call, every 10 seconds
29
+ beat: []
30
+ },
31
+ dead_max_jobs: 10_000,
32
+ dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
33
+ reloader: proc { |&block| block.call }
34
+ }
35
+
36
+ ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
37
+ l = cfg.logger
38
+ l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
39
+ l.warn("#{ex.class.name}: #{ex.message}")
40
+ l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
41
+ }
42
+
43
+ def initialize(options = {})
44
+ @options = DEFAULTS.merge(options)
45
+ @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
46
+ @directory = {}
47
+ @redis_config = {}
48
+ @capsules = {}
49
+ end
50
+
51
+ def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
52
+ attr_reader :options
53
+ attr_reader :capsules
54
+
55
+ # LEGACY: edits the default capsule
56
+ # config.concurrency = 5
57
+ def concurrency=(val)
58
+ default_capsule.concurrency = Integer(val)
59
+ end
60
+
61
+ def concurrency
62
+ default_capsule.concurrency
63
+ end
64
+
65
+ def total_concurrency
66
+ capsules.each_value.sum(&:concurrency)
67
+ end
68
+
69
+ # Edit the default capsule.
70
+ # config.queues = %w( high default low ) # strict
71
+ # config.queues = %w( high,3 default,2 low,1 ) # weighted
72
+ # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
73
+ #
74
+ # With weighted priority, queue will be checked first (weight / total) of the time.
75
+ # high will be checked first (3/6) or 50% of the time.
76
+ # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
77
+ # are ridiculous and unnecessarily expensive. You can get random queue ordering
78
+ # by explicitly setting all weights to 1.
79
+ def queues=(val)
80
+ default_capsule.queues = val
81
+ end
82
+
83
+ def queues
84
+ default_capsule.queues
85
+ end
86
+
87
+ def client_middleware
88
+ @client_chain ||= Sidekiq::Middleware::Chain.new(self)
89
+ yield @client_chain if block_given?
90
+ @client_chain
91
+ end
92
+
93
+ def server_middleware
94
+ @server_chain ||= Sidekiq::Middleware::Chain.new(self)
95
+ yield @server_chain if block_given?
96
+ @server_chain
97
+ end
98
+
99
+ def default_capsule(&block)
100
+ capsule("default", &block)
101
+ end
102
+
103
+ # register a new queue processing subsystem
104
+ def capsule(name)
105
+ nm = name.to_s
106
+ cap = @capsules.fetch(nm) do
107
+ cap = Sidekiq::Capsule.new(nm, self)
108
+ @capsules[nm] = cap
109
+ end
110
+ yield cap if block_given?
111
+ cap
112
+ end
113
+
114
+ # All capsules must use the same Redis configuration
115
+ def redis=(hash)
116
+ @redis_config = @redis_config.merge(hash)
117
+ end
118
+
119
+ def redis_pool
120
+ Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
121
+ end
122
+
123
+ private def local_redis_pool
124
+ # this is our internal client/housekeeping pool. each capsule has its
125
+ # own pool for executing threads.
126
+ @redis ||= new_redis_pool(5, "internal")
127
+ end
128
+
129
+ def new_redis_pool(size, name = "unset")
130
+ # connection pool is lazy, it will not create connections unless you actually need them
131
+ # so don't be skimpy!
132
+ RedisConnection.create(@redis_config.merge(size: size, logger: logger, pool_name: name))
133
+ end
134
+
135
+ def redis_info
136
+ redis do |conn|
137
+ conn.info
138
+ rescue RedisClientAdapter::CommandError => ex
139
+ # 2850 return fake version when INFO command has (probably) been renamed
140
+ raise unless /unknown command/.match?(ex.message)
141
+ {
142
+ "redis_version" => "9.9.9",
143
+ "uptime_in_days" => "9999",
144
+ "connected_clients" => "9999",
145
+ "used_memory_human" => "9P",
146
+ "used_memory_peak_human" => "9P"
147
+ }.freeze
148
+ end
149
+ end
150
+
151
+ def redis
152
+ raise ArgumentError, "requires a block" unless block_given?
153
+ redis_pool.with do |conn|
154
+ retryable = true
155
+ begin
156
+ yield conn
157
+ rescue RedisClientAdapter::BaseError => ex
158
+ # 2550 Failover can cause the server to become a replica, need
159
+ # to disconnect and reopen the socket to get back to the primary.
160
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
161
+ # 4985 Use the same logic when a blocking command is force-unblocked
162
+ # The same retry logic is also used in client.rb
163
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
164
+ conn.close
165
+ retryable = false
166
+ retry
167
+ end
168
+ raise
169
+ end
170
+ end
171
+ end
172
+
173
+ # register global singletons which can be accessed elsewhere
174
+ def register(name, instance)
175
+ @directory[name] = instance
176
+ end
177
+
178
+ # find a singleton
179
+ def lookup(name, default_class = nil)
180
+ # JNDI is just a fancy name for a hash lookup
181
+ @directory.fetch(name) do |key|
182
+ return nil unless default_class
183
+ @directory[key] = default_class.new(self)
184
+ end
185
+ end
186
+
187
+ ##
188
+ # Death handlers are called when all retries for a job have been exhausted and
189
+ # the job dies. It's the notification to your application
190
+ # that this job will not succeed without manual intervention.
191
+ #
192
+ # Sidekiq.configure_server do |config|
193
+ # config.death_handlers << ->(job, ex) do
194
+ # end
195
+ # end
196
+ def death_handlers
197
+ @options[:death_handlers]
198
+ end
199
+
200
+ # How frequently Redis should be checked by a random Sidekiq process for
201
+ # scheduled and retriable jobs. Each individual process will take turns by
202
+ # waiting some multiple of this value.
203
+ #
204
+ # See sidekiq/scheduled.rb for an in-depth explanation of this value
205
+ def average_scheduled_poll_interval=(interval)
206
+ @options[:average_scheduled_poll_interval] = interval
207
+ end
208
+
209
+ # Register a proc to handle any error which occurs within the Sidekiq process.
210
+ #
211
+ # Sidekiq.configure_server do |config|
212
+ # config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
213
+ # end
214
+ #
215
+ # The default error handler logs errors to @logger.
216
+ def error_handlers
217
+ @options[:error_handlers]
218
+ end
219
+
220
+ # Register a block to run at a point in the Sidekiq lifecycle.
221
+ # :startup, :quiet or :shutdown are valid events.
222
+ #
223
+ # Sidekiq.configure_server do |config|
224
+ # config.on(:shutdown) do
225
+ # puts "Goodbye cruel world!"
226
+ # end
227
+ # end
228
+ def on(event, &block)
229
+ raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
230
+ raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
231
+ @options[:lifecycle_events][event] << block
232
+ end
233
+
234
+ def logger
235
+ @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
236
+ log.level = Logger::INFO
237
+ log.formatter = if ENV["DYNO"]
238
+ Sidekiq::Logger::Formatters::WithoutTimestamp.new
239
+ else
240
+ Sidekiq::Logger::Formatters::Pretty.new
241
+ end
242
+ end
243
+ end
244
+
245
+ def logger=(logger)
246
+ if logger.nil?
247
+ self.logger.level = Logger::FATAL
248
+ return
249
+ end
250
+
251
+ logger.extend(Sidekiq::LoggingUtils)
252
+ @logger = logger
253
+ end
254
+
255
+ # INTERNAL USE ONLY
256
+ def handle_exception(ex, ctx = {})
257
+ if @options[:error_handlers].size == 0
258
+ p ["!!!!!", ex]
259
+ end
260
+ @options[:error_handlers].each do |handler|
261
+ handler.call(ex, ctx, self)
262
+ rescue => e
263
+ l = logger
264
+ l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
265
+ l.error e
266
+ l.error e.backtrace.join("\n") unless e.backtrace.nil?
267
+ end
268
+ end
269
+ end
270
+ end
@@ -0,0 +1,62 @@
1
+ require "sidekiq/redis_connection"
2
+ require "time"
3
+
4
+ # This file is designed to be required within the user's
5
+ # deployment script; it should need a bare minimum of dependencies.
6
+ # Usage:
7
+ #
8
+ # require "sidekiq/deploy"
9
+ # Sidekiq::Deploy.mark!("Some change")
10
+ #
11
+ # If you do not pass a label, Sidekiq will try to use the latest
12
+ # git commit info.
13
+ #
14
+
15
+ module Sidekiq
16
+ class Deploy
17
+ MARK_TTL = 90 * 24 * 60 * 60 # 90 days
18
+
19
+ LABEL_MAKER = -> {
20
+ `git log -1 --format="%h %s"`.strip
21
+ }
22
+
23
+ def self.mark!(label = nil)
24
+ label ||= LABEL_MAKER.call
25
+ Sidekiq::Deploy.new.mark(label: label)
26
+ end
27
+
28
+ def initialize(pool = Sidekiq::RedisConnection.create)
29
+ @pool = pool
30
+ end
31
+
32
+ def mark(at: Time.now, label: "")
33
+ # we need to round the timestamp so that we gracefully
34
+ # handle an very common error in marking deploys:
35
+ # having every process mark its deploy, leading
36
+ # to N marks for each deploy. Instead we round the time
37
+ # to the minute so that multple marks within that minute
38
+ # will all naturally rollup into one mark per minute.
39
+ whence = at.utc
40
+ floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
41
+ datecode = floor.strftime("%Y%m%d")
42
+ key = "#{datecode}-marks"
43
+ stamp = floor.iso8601
44
+
45
+ @pool.with do |c|
46
+ # only allow one deploy mark for a given label for the next minute
47
+ lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
48
+ if lock
49
+ c.multi do |pipe|
50
+ pipe.hsetnx(key, stamp, label)
51
+ pipe.expire(key, MARK_TTL)
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ def fetch(date = Time.now.utc.to_date)
58
+ datecode = date.strftime("%Y%m%d")
59
+ @pool.with { |c| c.hgetall("#{datecode}-marks") }
60
+ end
61
+ end
62
+ end