sidekiq 6.5.7 → 7.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (89) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +10 -4
  3. data/{LICENSE → LICENSE.txt} +0 -0
  4. data/README.md +13 -12
  5. data/bin/sidekiq +3 -8
  6. data/bin/sidekiqload +14 -23
  7. data/lib/sidekiq/api.rb +42 -111
  8. data/lib/sidekiq/capsule.rb +110 -0
  9. data/lib/sidekiq/cli.rb +44 -47
  10. data/lib/sidekiq/client.rb +29 -16
  11. data/lib/sidekiq/component.rb +1 -0
  12. data/lib/sidekiq/config.rb +270 -0
  13. data/lib/sidekiq/deploy.rb +62 -0
  14. data/lib/sidekiq/embedded.rb +61 -0
  15. data/lib/sidekiq/fetch.rb +10 -11
  16. data/lib/sidekiq/job.rb +375 -10
  17. data/lib/sidekiq/job_logger.rb +1 -1
  18. data/lib/sidekiq/job_retry.rb +8 -8
  19. data/lib/sidekiq/job_util.rb +4 -4
  20. data/lib/sidekiq/launcher.rb +36 -46
  21. data/lib/sidekiq/logger.rb +1 -26
  22. data/lib/sidekiq/manager.rb +9 -11
  23. data/lib/sidekiq/metrics/query.rb +2 -2
  24. data/lib/sidekiq/metrics/shared.rb +4 -3
  25. data/lib/sidekiq/metrics/tracking.rb +18 -18
  26. data/lib/sidekiq/middleware/chain.rb +7 -9
  27. data/lib/sidekiq/middleware/current_attributes.rb +8 -13
  28. data/lib/sidekiq/pool.rb +7 -0
  29. data/lib/sidekiq/processor.rb +17 -26
  30. data/lib/sidekiq/redis_client_adapter.rb +9 -45
  31. data/lib/sidekiq/redis_connection.rb +11 -111
  32. data/lib/sidekiq/scheduled.rb +19 -20
  33. data/lib/sidekiq/testing.rb +4 -32
  34. data/lib/sidekiq/transaction_aware_client.rb +4 -5
  35. data/lib/sidekiq/version.rb +1 -1
  36. data/lib/sidekiq/web/application.rb +1 -1
  37. data/lib/sidekiq/web/csrf_protection.rb +1 -1
  38. data/lib/sidekiq/web/helpers.rb +4 -15
  39. data/lib/sidekiq/web.rb +2 -17
  40. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  41. data/lib/sidekiq.rb +52 -274
  42. data/sidekiq.gemspec +29 -5
  43. data/web/assets/javascripts/base-charts.js +106 -0
  44. data/web/assets/javascripts/dashboard-charts.js +166 -0
  45. data/web/assets/javascripts/dashboard.js +3 -223
  46. data/web/assets/javascripts/metrics.js +90 -116
  47. data/web/assets/stylesheets/application-rtl.css +2 -91
  48. data/web/assets/stylesheets/application.css +21 -296
  49. data/web/locales/ar.yml +70 -70
  50. data/web/locales/cs.yml +62 -62
  51. data/web/locales/da.yml +52 -52
  52. data/web/locales/de.yml +65 -65
  53. data/web/locales/el.yml +2 -7
  54. data/web/locales/en.yml +76 -70
  55. data/web/locales/es.yml +68 -68
  56. data/web/locales/fa.yml +65 -65
  57. data/web/locales/fr.yml +67 -67
  58. data/web/locales/he.yml +65 -64
  59. data/web/locales/hi.yml +59 -59
  60. data/web/locales/it.yml +53 -53
  61. data/web/locales/ja.yml +64 -68
  62. data/web/locales/ko.yml +52 -52
  63. data/web/locales/lt.yml +66 -66
  64. data/web/locales/nb.yml +61 -61
  65. data/web/locales/nl.yml +52 -52
  66. data/web/locales/pl.yml +45 -45
  67. data/web/locales/pt-br.yml +59 -69
  68. data/web/locales/pt.yml +51 -51
  69. data/web/locales/ru.yml +67 -66
  70. data/web/locales/sv.yml +53 -53
  71. data/web/locales/ta.yml +60 -60
  72. data/web/locales/uk.yml +62 -61
  73. data/web/locales/ur.yml +64 -64
  74. data/web/locales/vi.yml +67 -67
  75. data/web/locales/zh-cn.yml +1 -0
  76. data/web/locales/zh-tw.yml +10 -1
  77. data/web/views/_footer.erb +5 -2
  78. data/web/views/dashboard.erb +36 -5
  79. data/web/views/metrics.erb +30 -19
  80. data/web/views/metrics_for_job.erb +16 -34
  81. metadata +59 -30
  82. data/lib/sidekiq/delay.rb +0 -43
  83. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  84. data/lib/sidekiq/extensions/active_record.rb +0 -43
  85. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  86. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  87. data/lib/sidekiq/metrics/deploy.rb +0 -47
  88. data/lib/sidekiq/worker.rb +0 -370
  89. data/web/assets/javascripts/graph.js +0 -16
data/lib/sidekiq/cli.rb CHANGED
@@ -9,7 +9,9 @@ require "erb"
9
9
  require "fileutils"
10
10
 
11
11
  require "sidekiq"
12
+ require "sidekiq/config"
12
13
  require "sidekiq/component"
14
+ require "sidekiq/capsule"
13
15
  require "sidekiq/launcher"
14
16
 
15
17
  module Sidekiq # :nodoc:
@@ -22,9 +24,7 @@ module Sidekiq # :nodoc:
22
24
  attr_accessor :config
23
25
 
24
26
  def parse(args = ARGV.dup)
25
- @config = Sidekiq
26
- @config[:error_handlers].clear
27
- @config[:error_handlers] << @config.method(:default_error_handler)
27
+ @config ||= Sidekiq.default_configuration
28
28
 
29
29
  setup_options(args)
30
30
  initialize_logger
@@ -41,7 +41,7 @@ module Sidekiq # :nodoc:
41
41
  def run(boot_app: true)
42
42
  boot_application if boot_app
43
43
 
44
- if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
44
+ if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
45
45
  print_banner
46
46
  end
47
47
  logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -73,8 +73,8 @@ module Sidekiq # :nodoc:
73
73
  # touch the connection pool so it is created before we
74
74
  # fire startup and start multithreading.
75
75
  info = @config.redis_info
76
- ver = info["redis_version"]
77
- raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
76
+ ver = Gem::Version.new(info["redis_version"])
77
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
78
78
 
79
79
  maxmemory_policy = info["maxmemory_policy"]
80
80
  if maxmemory_policy != "noeviction"
@@ -90,9 +90,9 @@ module Sidekiq # :nodoc:
90
90
 
91
91
  # Since the user can pass us a connection pool explicitly in the initializer, we
92
92
  # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
93
- cursize = @config.redis_pool.size
94
- needed = @config[:concurrency] + 2
95
- raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
93
+ @config.capsules.each_pair do |name, cap|
94
+ raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
95
+ end
96
96
 
97
97
  # cache process identity
98
98
  @config[:identity] = identity
@@ -104,8 +104,8 @@ module Sidekiq # :nodoc:
104
104
  # Starting here the process will now have multiple threads running.
105
105
  fire_event(:startup, reverse: false, reraise: true)
106
106
 
107
- logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
108
- logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
107
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
108
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
109
109
 
110
110
  launch(self_read)
111
111
  end
@@ -138,19 +138,34 @@ module Sidekiq # :nodoc:
138
138
  end
139
139
  end
140
140
 
141
- def self.w
142
- "\e[37m"
141
+ HOLIDAY_COLORS = {
142
+ # got other color-specific holidays from around the world?
143
+ # https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
144
+ "3-17" => "\e[1;32m", # St. Patrick's Day green
145
+ "10-31" => "\e[38;5;208m" # Halloween orange
146
+ }
147
+
148
+ def self.day
149
+ @@day ||= begin
150
+ t = Date.today
151
+ "#{t.month}-#{t.day}"
152
+ end
143
153
  end
144
154
 
145
155
  def self.r
146
- "\e[31m"
156
+ @@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
147
157
  end
148
158
 
149
159
  def self.b
150
- "\e[30m"
160
+ @@b ||= HOLIDAY_COLORS[day] || "\e[30m"
161
+ end
162
+
163
+ def self.w
164
+ "\e[1;37m"
151
165
  end
152
166
 
153
167
  def self.reset
168
+ @@b = @@r = @@day = nil
154
169
  "\e[0m"
155
170
  end
156
171
 
@@ -163,7 +178,7 @@ module Sidekiq # :nodoc:
163
178
  #{w} ,$$$$$b#{b}/#{w}md$$$P^'
164
179
  #{w} .d$$$$$$#{b}/#{w}$$$P'
165
180
  #{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
166
- #{w} $: ,$$: #{r} / ___|(_) __| | ___| | _(_) __ _
181
+ #{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
167
182
  #{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
168
183
  #{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
169
184
  #{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
@@ -260,6 +275,11 @@ module Sidekiq # :nodoc:
260
275
 
261
276
  # merge with defaults
262
277
  @config.merge!(opts)
278
+
279
+ @config.default_capsule.tap do |cap|
280
+ cap.queues = opts[:queues]
281
+ cap.concurrency = opts[:concurrency] || @config[:concurrency]
282
+ end
263
283
  end
264
284
 
265
285
  def boot_application
@@ -267,12 +287,11 @@ module Sidekiq # :nodoc:
267
287
 
268
288
  if File.directory?(@config[:require])
269
289
  require "rails"
270
- if ::Rails::VERSION::MAJOR < 5
271
- raise "Sidekiq no longer supports this version of Rails"
272
- else
273
- require "sidekiq/rails"
274
- require File.expand_path("#{@config[:require]}/config/environment.rb")
290
+ if ::Rails::VERSION::MAJOR < 6
291
+ warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
275
292
  end
293
+ require "sidekiq/rails"
294
+ require File.expand_path("#{@config[:require]}/config/environment.rb")
276
295
  @config[:tag] ||= default_tag
277
296
  else
278
297
  require @config[:require]
@@ -333,8 +352,8 @@ module Sidekiq # :nodoc:
333
352
  end
334
353
 
335
354
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
336
- queue, weight = arg.split(",")
337
- parse_queue opts, queue, weight
355
+ opts[:queues] ||= []
356
+ opts[:queues] << arg
338
357
  end
339
358
 
340
359
  o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
@@ -383,7 +402,7 @@ module Sidekiq # :nodoc:
383
402
  def parse_config(path)
384
403
  erb = ERB.new(File.read(path))
385
404
  erb.filename = File.expand_path(path)
386
- opts = load_yaml(erb.result) || {}
405
+ opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
387
406
 
388
407
  if opts.respond_to? :deep_symbolize_keys!
389
408
  opts.deep_symbolize_keys!
@@ -394,31 +413,9 @@ module Sidekiq # :nodoc:
394
413
  opts = opts.merge(opts.delete(environment.to_sym) || {})
395
414
  opts.delete(:strict)
396
415
 
397
- parse_queues(opts, opts.delete(:queues) || [])
398
-
399
416
  opts
400
417
  end
401
418
 
402
- def load_yaml(src)
403
- if Psych::VERSION > "4.0"
404
- YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
405
- else
406
- YAML.load(src)
407
- end
408
- end
409
-
410
- def parse_queues(opts, queues_and_weights)
411
- queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
412
- end
413
-
414
- def parse_queue(opts, queue, weight = nil)
415
- opts[:queues] ||= []
416
- opts[:strict] = true if opts[:strict].nil?
417
- raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
418
- [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
419
- opts[:strict] = false if weight.to_i > 0
420
- end
421
-
422
419
  def rails_app?
423
420
  defined?(::Rails) && ::Rails.respond_to?(:application)
424
421
  end
@@ -426,4 +423,4 @@ module Sidekiq # :nodoc:
426
423
  end
427
424
 
428
425
  require "sidekiq/systemd"
429
- require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
426
+ require "sidekiq/metrics/tracking"
@@ -21,7 +21,6 @@ module Sidekiq
21
21
  # Sidekiq.client_middleware but you can change as necessary.
22
22
  #
23
23
  def middleware(&block)
24
- @chain ||= Sidekiq.client_middleware
25
24
  if block
26
25
  @chain = @chain.dup
27
26
  yield @chain
@@ -31,18 +30,32 @@ module Sidekiq
31
30
 
32
31
  attr_accessor :redis_pool
33
32
 
34
- # Sidekiq::Client normally uses the default Redis pool but you may
35
- # pass a custom ConnectionPool if you want to shard your
36
- # Sidekiq jobs across several Redis instances (for scalability
37
- # reasons, e.g.)
33
+ # Sidekiq::Client is responsible for pushing job payloads to Redis.
34
+ # Requires the :pool or :config keyword argument.
38
35
  #
39
- # Sidekiq::Client.new(ConnectionPool.new { Redis.new })
36
+ # Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
40
37
  #
41
- # Generally this is only needed for very large Sidekiq installs processing
42
- # thousands of jobs per second. I don't recommend sharding unless you
43
- # cannot scale any other way (e.g. splitting your app into smaller apps).
44
- def initialize(redis_pool = nil)
45
- @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
38
+ # Inside the Sidekiq process, you can reuse the configured resources:
39
+ #
40
+ # Sidekiq::Client.new(config: config)
41
+ #
42
+ # @param pool [ConnectionPool] explicit Redis pool to use
43
+ # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
44
+ # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
45
+ def initialize(*args, **kwargs)
46
+ if args.size == 1 && kwargs.size == 0
47
+ warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
48
+ # old calling method, accept 1 pool argument
49
+ @redis_pool = args[0]
50
+ @chain = Sidekiq.default_configuration.client_middleware
51
+ @config = Sidekiq.default_configuration
52
+ else
53
+ # new calling method: keyword arguments
54
+ @config = kwargs[:config] || Sidekiq.default_configuration
55
+ @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
56
+ @chain = kwargs[:chain] || @config&.client_middleware
57
+ raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
58
+ end
46
59
  end
47
60
 
48
61
  ##
@@ -135,11 +148,11 @@ module Sidekiq
135
148
  # you cannot scale any other way (e.g. splitting your app into smaller apps).
136
149
  def self.via(pool)
137
150
  raise ArgumentError, "No pool given" if pool.nil?
138
- current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
139
- Thread.current[:sidekiq_via_pool] = pool
151
+ current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
152
+ Thread.current[:sidekiq_redis_pool] = pool
140
153
  yield
141
154
  ensure
142
- Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
155
+ Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
143
156
  end
144
157
 
145
158
  class << self
@@ -201,14 +214,14 @@ module Sidekiq
201
214
  conn.pipelined do |pipeline|
202
215
  atomic_push(pipeline, payloads)
203
216
  end
204
- rescue RedisConnection.adapter::BaseError => ex
217
+ rescue RedisClient::Error => ex
205
218
  # 2550 Failover can cause the server to become a replica, need
206
219
  # to disconnect and reopen the socket to get back to the primary.
207
220
  # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
221
  # 4985 Use the same logic when a blocking command is force-unblocked
209
222
  # The retry logic is copied from sidekiq.rb
210
223
  if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
- conn.disconnect!
224
+ conn.close
212
225
  retryable = false
213
226
  retry
214
227
  end
@@ -50,6 +50,7 @@ module Sidekiq
50
50
  oneshot = options.fetch(:oneshot, true)
51
51
  reverse = options[:reverse]
52
52
  reraise = options[:reraise]
53
+ logger.debug("Firing #{event} event") if oneshot
53
54
 
54
55
  arr = config[:lifecycle_events][event]
55
56
  arr.reverse! if reverse
@@ -0,0 +1,270 @@
1
+ require "forwardable"
2
+
3
+ require "set"
4
+ require "sidekiq/redis_connection"
5
+
6
+ module Sidekiq
7
+ # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
8
+ class Config
9
+ extend Forwardable
10
+
11
+ DEFAULTS = {
12
+ labels: Set.new,
13
+ require: ".",
14
+ environment: nil,
15
+ concurrency: 5,
16
+ timeout: 25,
17
+ poll_interval_average: nil,
18
+ average_scheduled_poll_interval: 5,
19
+ on_complex_arguments: :raise,
20
+ error_handlers: [],
21
+ death_handlers: [],
22
+ lifecycle_events: {
23
+ startup: [],
24
+ quiet: [],
25
+ shutdown: [],
26
+ # triggers when we fire the first heartbeat on startup OR repairing a network partition
27
+ heartbeat: [],
28
+ # triggers on EVERY heartbeat call, every 10 seconds
29
+ beat: []
30
+ },
31
+ dead_max_jobs: 10_000,
32
+ dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
33
+ reloader: proc { |&block| block.call }
34
+ }
35
+
36
+ ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
37
+ l = cfg.logger
38
+ l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
39
+ l.warn("#{ex.class.name}: #{ex.message}")
40
+ l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
41
+ }
42
+
43
+ def initialize(options = {})
44
+ @options = DEFAULTS.merge(options)
45
+ @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
46
+ @directory = {}
47
+ @redis_config = {}
48
+ @capsules = {}
49
+ end
50
+
51
+ def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
52
+ attr_reader :options
53
+ attr_reader :capsules
54
+
55
+ # LEGACY: edits the default capsule
56
+ # config.concurrency = 5
57
+ def concurrency=(val)
58
+ default_capsule.concurrency = Integer(val)
59
+ end
60
+
61
+ def concurrency
62
+ default_capsule.concurrency
63
+ end
64
+
65
+ def total_concurrency
66
+ capsules.each_value.sum(&:concurrency)
67
+ end
68
+
69
+ # Edit the default capsule.
70
+ # config.queues = %w( high default low ) # strict
71
+ # config.queues = %w( high,3 default,2 low,1 ) # weighted
72
+ # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
73
+ #
74
+ # With weighted priority, queue will be checked first (weight / total) of the time.
75
+ # high will be checked first (3/6) or 50% of the time.
76
+ # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
77
+ # are ridiculous and unnecessarily expensive. You can get random queue ordering
78
+ # by explicitly setting all weights to 1.
79
+ def queues=(val)
80
+ default_capsule.queues = val
81
+ end
82
+
83
+ def queues
84
+ default_capsule.queues
85
+ end
86
+
87
+ def client_middleware
88
+ @client_chain ||= Sidekiq::Middleware::Chain.new(self)
89
+ yield @client_chain if block_given?
90
+ @client_chain
91
+ end
92
+
93
+ def server_middleware
94
+ @server_chain ||= Sidekiq::Middleware::Chain.new(self)
95
+ yield @server_chain if block_given?
96
+ @server_chain
97
+ end
98
+
99
+ def default_capsule(&block)
100
+ capsule("default", &block)
101
+ end
102
+
103
+ # register a new queue processing subsystem
104
+ def capsule(name)
105
+ nm = name.to_s
106
+ cap = @capsules.fetch(nm) do
107
+ cap = Sidekiq::Capsule.new(nm, self)
108
+ @capsules[nm] = cap
109
+ end
110
+ yield cap if block_given?
111
+ cap
112
+ end
113
+
114
+ # All capsules must use the same Redis configuration
115
+ def redis=(hash)
116
+ @redis_config = @redis_config.merge(hash)
117
+ end
118
+
119
+ def redis_pool
120
+ Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
121
+ end
122
+
123
+ private def local_redis_pool
124
+ # this is our internal client/housekeeping pool. each capsule has its
125
+ # own pool for executing threads.
126
+ @redis ||= new_redis_pool(5, "internal")
127
+ end
128
+
129
+ def new_redis_pool(size, name = "unset")
130
+ # connection pool is lazy, it will not create connections unless you actually need them
131
+ # so don't be skimpy!
132
+ RedisConnection.create(@redis_config.merge(size: size, logger: logger, pool_name: name))
133
+ end
134
+
135
+ def redis_info
136
+ redis do |conn|
137
+ conn.info
138
+ rescue RedisClientAdapter::CommandError => ex
139
+ # 2850 return fake version when INFO command has (probably) been renamed
140
+ raise unless /unknown command/.match?(ex.message)
141
+ {
142
+ "redis_version" => "9.9.9",
143
+ "uptime_in_days" => "9999",
144
+ "connected_clients" => "9999",
145
+ "used_memory_human" => "9P",
146
+ "used_memory_peak_human" => "9P"
147
+ }.freeze
148
+ end
149
+ end
150
+
151
+ def redis
152
+ raise ArgumentError, "requires a block" unless block_given?
153
+ redis_pool.with do |conn|
154
+ retryable = true
155
+ begin
156
+ yield conn
157
+ rescue RedisClientAdapter::BaseError => ex
158
+ # 2550 Failover can cause the server to become a replica, need
159
+ # to disconnect and reopen the socket to get back to the primary.
160
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
161
+ # 4985 Use the same logic when a blocking command is force-unblocked
162
+ # The same retry logic is also used in client.rb
163
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
164
+ conn.close
165
+ retryable = false
166
+ retry
167
+ end
168
+ raise
169
+ end
170
+ end
171
+ end
172
+
173
+ # register global singletons which can be accessed elsewhere
174
+ def register(name, instance)
175
+ @directory[name] = instance
176
+ end
177
+
178
+ # find a singleton
179
+ def lookup(name, default_class = nil)
180
+ # JNDI is just a fancy name for a hash lookup
181
+ @directory.fetch(name) do |key|
182
+ return nil unless default_class
183
+ @directory[key] = default_class.new(self)
184
+ end
185
+ end
186
+
187
+ ##
188
+ # Death handlers are called when all retries for a job have been exhausted and
189
+ # the job dies. It's the notification to your application
190
+ # that this job will not succeed without manual intervention.
191
+ #
192
+ # Sidekiq.configure_server do |config|
193
+ # config.death_handlers << ->(job, ex) do
194
+ # end
195
+ # end
196
+ def death_handlers
197
+ @options[:death_handlers]
198
+ end
199
+
200
+ # How frequently Redis should be checked by a random Sidekiq process for
201
+ # scheduled and retriable jobs. Each individual process will take turns by
202
+ # waiting some multiple of this value.
203
+ #
204
+ # See sidekiq/scheduled.rb for an in-depth explanation of this value
205
+ def average_scheduled_poll_interval=(interval)
206
+ @options[:average_scheduled_poll_interval] = interval
207
+ end
208
+
209
+ # Register a proc to handle any error which occurs within the Sidekiq process.
210
+ #
211
+ # Sidekiq.configure_server do |config|
212
+ # config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
213
+ # end
214
+ #
215
+ # The default error handler logs errors to @logger.
216
+ def error_handlers
217
+ @options[:error_handlers]
218
+ end
219
+
220
+ # Register a block to run at a point in the Sidekiq lifecycle.
221
+ # :startup, :quiet or :shutdown are valid events.
222
+ #
223
+ # Sidekiq.configure_server do |config|
224
+ # config.on(:shutdown) do
225
+ # puts "Goodbye cruel world!"
226
+ # end
227
+ # end
228
+ def on(event, &block)
229
+ raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
230
+ raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
231
+ @options[:lifecycle_events][event] << block
232
+ end
233
+
234
+ def logger
235
+ @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
236
+ log.level = Logger::INFO
237
+ log.formatter = if ENV["DYNO"]
238
+ Sidekiq::Logger::Formatters::WithoutTimestamp.new
239
+ else
240
+ Sidekiq::Logger::Formatters::Pretty.new
241
+ end
242
+ end
243
+ end
244
+
245
+ def logger=(logger)
246
+ if logger.nil?
247
+ self.logger.level = Logger::FATAL
248
+ return
249
+ end
250
+
251
+ logger.extend(Sidekiq::LoggingUtils)
252
+ @logger = logger
253
+ end
254
+
255
+ # INTERNAL USE ONLY
256
+ def handle_exception(ex, ctx = {})
257
+ if @options[:error_handlers].size == 0
258
+ p ["!!!!!", ex]
259
+ end
260
+ @options[:error_handlers].each do |handler|
261
+ handler.call(ex, ctx, self)
262
+ rescue => e
263
+ l = logger
264
+ l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
265
+ l.error e
266
+ l.error e.backtrace.join("\n") unless e.backtrace.nil?
267
+ end
268
+ end
269
+ end
270
+ end
@@ -0,0 +1,62 @@
1
+ require "sidekiq/redis_connection"
2
+ require "time"
3
+
4
+ # This file is designed to be required within the user's
5
+ # deployment script; it should need a bare minimum of dependencies.
6
+ # Usage:
7
+ #
8
+ # require "sidekiq/deploy"
9
+ # Sidekiq::Deploy.mark!("Some change")
10
+ #
11
+ # If you do not pass a label, Sidekiq will try to use the latest
12
+ # git commit info.
13
+ #
14
+
15
+ module Sidekiq
16
+ class Deploy
17
+ MARK_TTL = 90 * 24 * 60 * 60 # 90 days
18
+
19
+ LABEL_MAKER = -> {
20
+ `git log -1 --format="%h %s"`.strip
21
+ }
22
+
23
+ def self.mark!(label = nil)
24
+ label ||= LABEL_MAKER.call
25
+ Sidekiq::Deploy.new.mark(label: label)
26
+ end
27
+
28
+ def initialize(pool = Sidekiq::RedisConnection.create)
29
+ @pool = pool
30
+ end
31
+
32
+ def mark(at: Time.now, label: "")
33
+ # we need to round the timestamp so that we gracefully
34
+ # handle an very common error in marking deploys:
35
+ # having every process mark its deploy, leading
36
+ # to N marks for each deploy. Instead we round the time
37
+ # to the minute so that multple marks within that minute
38
+ # will all naturally rollup into one mark per minute.
39
+ whence = at.utc
40
+ floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
41
+ datecode = floor.strftime("%Y%m%d")
42
+ key = "#{datecode}-marks"
43
+ stamp = floor.iso8601
44
+
45
+ @pool.with do |c|
46
+ # only allow one deploy mark for a given label for the next minute
47
+ lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
48
+ if lock
49
+ c.multi do |pipe|
50
+ pipe.hsetnx(key, stamp, label)
51
+ pipe.expire(key, MARK_TTL)
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ def fetch(date = Time.now.utc.to_date)
58
+ datecode = date.strftime("%Y%m%d")
59
+ @pool.with { |c| c.hgetall("#{datecode}-marks") }
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,61 @@
1
+ require "sidekiq/component"
2
+ require "sidekiq/launcher"
3
+ require "sidekiq/metrics/tracking"
4
+
5
+ module Sidekiq
6
+ class Embedded
7
+ include Sidekiq::Component
8
+
9
+ def initialize(config)
10
+ @config = config
11
+ end
12
+
13
+ def run
14
+ housekeeping
15
+ fire_event(:startup, reverse: false, reraise: true)
16
+ @launcher = Sidekiq::Launcher.new(@config, embedded: true)
17
+ @launcher.run
18
+ sleep 0.1 # pause to give threads time to spin up
19
+
20
+ logger.info "Embedded mode running with #{Thread.list.size} threads"
21
+ logger.debug { Thread.list.map(&:name) }
22
+ end
23
+
24
+ def quiet
25
+ @launcher&.quiet
26
+ end
27
+
28
+ def stop
29
+ @launcher&.stop
30
+ end
31
+
32
+ private
33
+
34
+ def housekeeping
35
+ logger.info "Running in #{RUBY_DESCRIPTION}"
36
+ logger.info Sidekiq::LICENSE
37
+ logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
38
+
39
+ # touch the connection pool so it is created before we
40
+ # fire startup and start multithreading.
41
+ info = config.redis_info
42
+ ver = Gem::Version.new(info["redis_version"])
43
+ raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
44
+
45
+ maxmemory_policy = info["maxmemory_policy"]
46
+ if maxmemory_policy != "noeviction"
47
+ logger.warn <<~EOM
48
+
49
+
50
+ WARNING: Your Redis instance will evict Sidekiq data under heavy load.
51
+ The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
52
+ See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
53
+
54
+ EOM
55
+ end
56
+
57
+ logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
58
+ logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
59
+ end
60
+ end
61
+ end