sidekiq 6.5.6 → 7.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +16 -4
- data/{LICENSE → LICENSE.txt} +0 -0
- data/README.md +13 -12
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +15 -24
- data/lib/sidekiq/api.rb +55 -111
- data/lib/sidekiq/capsule.rb +110 -0
- data/lib/sidekiq/cli.rb +44 -47
- data/lib/sidekiq/client.rb +29 -16
- data/lib/sidekiq/component.rb +1 -0
- data/lib/sidekiq/config.rb +270 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +10 -11
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +8 -8
- data/lib/sidekiq/job_util.rb +4 -4
- data/lib/sidekiq/launcher.rb +36 -46
- data/lib/sidekiq/logger.rb +1 -26
- data/lib/sidekiq/manager.rb +9 -11
- data/lib/sidekiq/metrics/query.rb +2 -2
- data/lib/sidekiq/metrics/shared.rb +4 -3
- data/lib/sidekiq/metrics/tracking.rb +18 -18
- data/lib/sidekiq/middleware/chain.rb +7 -9
- data/lib/sidekiq/middleware/current_attributes.rb +8 -13
- data/lib/sidekiq/pool.rb +7 -0
- data/lib/sidekiq/processor.rb +17 -26
- data/lib/sidekiq/redis_client_adapter.rb +9 -45
- data/lib/sidekiq/redis_connection.rb +11 -111
- data/lib/sidekiq/scheduled.rb +58 -25
- data/lib/sidekiq/testing.rb +4 -32
- data/lib/sidekiq/transaction_aware_client.rb +4 -5
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +1 -1
- data/lib/sidekiq/web/csrf_protection.rb +1 -1
- data/lib/sidekiq/web/helpers.rb +5 -16
- data/lib/sidekiq/web.rb +2 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +52 -274
- data/sidekiq.gemspec +29 -5
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -223
- data/web/assets/javascripts/metrics.js +90 -116
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +21 -296
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +52 -52
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +2 -7
- data/web/locales/en.yml +76 -70
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +71 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +59 -69
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +37 -11
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/dashboard.erb +36 -5
- data/web/views/metrics.erb +30 -19
- data/web/views/metrics_for_job.erb +16 -34
- metadata +59 -30
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/metrics/deploy.rb +0 -47
- data/lib/sidekiq/worker.rb +0 -367
- data/web/assets/javascripts/graph.js +0 -16
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,7 +9,9 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
12
13
|
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
13
15
|
require "sidekiq/launcher"
|
14
16
|
|
15
17
|
module Sidekiq # :nodoc:
|
@@ -22,9 +24,7 @@ module Sidekiq # :nodoc:
|
|
22
24
|
attr_accessor :config
|
23
25
|
|
24
26
|
def parse(args = ARGV.dup)
|
25
|
-
@config
|
26
|
-
@config[:error_handlers].clear
|
27
|
-
@config[:error_handlers] << @config.method(:default_error_handler)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
28
28
|
|
29
29
|
setup_options(args)
|
30
30
|
initialize_logger
|
@@ -41,7 +41,7 @@ module Sidekiq # :nodoc:
|
|
41
41
|
def run(boot_app: true)
|
42
42
|
boot_application if boot_app
|
43
43
|
|
44
|
-
if environment == "development" && $stdout.tty? && @config.
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
45
|
print_banner
|
46
46
|
end
|
47
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -73,8 +73,8 @@ module Sidekiq # :nodoc:
|
|
73
73
|
# touch the connection pool so it is created before we
|
74
74
|
# fire startup and start multithreading.
|
75
75
|
info = @config.redis_info
|
76
|
-
ver = info["redis_version"]
|
77
|
-
raise "You are connecting to Redis
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
78
78
|
|
79
79
|
maxmemory_policy = info["maxmemory_policy"]
|
80
80
|
if maxmemory_policy != "noeviction"
|
@@ -90,9 +90,9 @@ module Sidekiq # :nodoc:
|
|
90
90
|
|
91
91
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
92
92
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
93
|
-
|
94
|
-
|
95
|
-
|
93
|
+
@config.capsules.each_pair do |name, cap|
|
94
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
95
|
+
end
|
96
96
|
|
97
97
|
# cache process identity
|
98
98
|
@config[:identity] = identity
|
@@ -104,8 +104,8 @@ module Sidekiq # :nodoc:
|
|
104
104
|
# Starting here the process will now have multiple threads running.
|
105
105
|
fire_event(:startup, reverse: false, reraise: true)
|
106
106
|
|
107
|
-
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
108
|
-
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
107
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
109
109
|
|
110
110
|
launch(self_read)
|
111
111
|
end
|
@@ -138,19 +138,34 @@ module Sidekiq # :nodoc:
|
|
138
138
|
end
|
139
139
|
end
|
140
140
|
|
141
|
-
|
142
|
-
|
141
|
+
HOLIDAY_COLORS = {
|
142
|
+
# got other color-specific holidays from around the world?
|
143
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
144
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
145
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
146
|
+
}
|
147
|
+
|
148
|
+
def self.day
|
149
|
+
@@day ||= begin
|
150
|
+
t = Date.today
|
151
|
+
"#{t.month}-#{t.day}"
|
152
|
+
end
|
143
153
|
end
|
144
154
|
|
145
155
|
def self.r
|
146
|
-
"\e[31m"
|
156
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
147
157
|
end
|
148
158
|
|
149
159
|
def self.b
|
150
|
-
"\e[30m"
|
160
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.w
|
164
|
+
"\e[1;37m"
|
151
165
|
end
|
152
166
|
|
153
167
|
def self.reset
|
168
|
+
@@b = @@r = @@day = nil
|
154
169
|
"\e[0m"
|
155
170
|
end
|
156
171
|
|
@@ -163,7 +178,7 @@ module Sidekiq # :nodoc:
|
|
163
178
|
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
164
179
|
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
165
180
|
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
166
|
-
#{w} $:
|
181
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
167
182
|
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
168
183
|
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
169
184
|
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
@@ -260,6 +275,11 @@ module Sidekiq # :nodoc:
|
|
260
275
|
|
261
276
|
# merge with defaults
|
262
277
|
@config.merge!(opts)
|
278
|
+
|
279
|
+
@config.default_capsule.tap do |cap|
|
280
|
+
cap.queues = opts[:queues]
|
281
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
282
|
+
end
|
263
283
|
end
|
264
284
|
|
265
285
|
def boot_application
|
@@ -267,12 +287,11 @@ module Sidekiq # :nodoc:
|
|
267
287
|
|
268
288
|
if File.directory?(@config[:require])
|
269
289
|
require "rails"
|
270
|
-
if ::Rails::VERSION::MAJOR <
|
271
|
-
|
272
|
-
else
|
273
|
-
require "sidekiq/rails"
|
274
|
-
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
290
|
+
if ::Rails::VERSION::MAJOR < 6
|
291
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
275
292
|
end
|
293
|
+
require "sidekiq/rails"
|
294
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
276
295
|
@config[:tag] ||= default_tag
|
277
296
|
else
|
278
297
|
require @config[:require]
|
@@ -333,8 +352,8 @@ module Sidekiq # :nodoc:
|
|
333
352
|
end
|
334
353
|
|
335
354
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
336
|
-
|
337
|
-
|
355
|
+
opts[:queues] ||= []
|
356
|
+
opts[:queues] << arg
|
338
357
|
end
|
339
358
|
|
340
359
|
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
@@ -383,7 +402,7 @@ module Sidekiq # :nodoc:
|
|
383
402
|
def parse_config(path)
|
384
403
|
erb = ERB.new(File.read(path))
|
385
404
|
erb.filename = File.expand_path(path)
|
386
|
-
opts =
|
405
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
387
406
|
|
388
407
|
if opts.respond_to? :deep_symbolize_keys!
|
389
408
|
opts.deep_symbolize_keys!
|
@@ -394,31 +413,9 @@ module Sidekiq # :nodoc:
|
|
394
413
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
395
414
|
opts.delete(:strict)
|
396
415
|
|
397
|
-
parse_queues(opts, opts.delete(:queues) || [])
|
398
|
-
|
399
416
|
opts
|
400
417
|
end
|
401
418
|
|
402
|
-
def load_yaml(src)
|
403
|
-
if Psych::VERSION > "4.0"
|
404
|
-
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
405
|
-
else
|
406
|
-
YAML.load(src)
|
407
|
-
end
|
408
|
-
end
|
409
|
-
|
410
|
-
def parse_queues(opts, queues_and_weights)
|
411
|
-
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
412
|
-
end
|
413
|
-
|
414
|
-
def parse_queue(opts, queue, weight = nil)
|
415
|
-
opts[:queues] ||= []
|
416
|
-
opts[:strict] = true if opts[:strict].nil?
|
417
|
-
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
418
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
419
|
-
opts[:strict] = false if weight.to_i > 0
|
420
|
-
end
|
421
|
-
|
422
419
|
def rails_app?
|
423
420
|
defined?(::Rails) && ::Rails.respond_to?(:application)
|
424
421
|
end
|
@@ -426,4 +423,4 @@ module Sidekiq # :nodoc:
|
|
426
423
|
end
|
427
424
|
|
428
425
|
require "sidekiq/systemd"
|
429
|
-
require "sidekiq/metrics/tracking"
|
426
|
+
require "sidekiq/metrics/tracking"
|
data/lib/sidekiq/client.rb
CHANGED
@@ -21,7 +21,6 @@ module Sidekiq
|
|
21
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
22
22
|
#
|
23
23
|
def middleware(&block)
|
24
|
-
@chain ||= Sidekiq.client_middleware
|
25
24
|
if block
|
26
25
|
@chain = @chain.dup
|
27
26
|
yield @chain
|
@@ -31,18 +30,32 @@ module Sidekiq
|
|
31
30
|
|
32
31
|
attr_accessor :redis_pool
|
33
32
|
|
34
|
-
# Sidekiq::Client
|
35
|
-
#
|
36
|
-
# Sidekiq jobs across several Redis instances (for scalability
|
37
|
-
# reasons, e.g.)
|
33
|
+
# Sidekiq::Client is responsible for pushing job payloads to Redis.
|
34
|
+
# Requires the :pool or :config keyword argument.
|
38
35
|
#
|
39
|
-
# Sidekiq::Client.new(
|
36
|
+
# Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
|
40
37
|
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
|
45
|
-
|
38
|
+
# Inside the Sidekiq process, you can reuse the configured resources:
|
39
|
+
#
|
40
|
+
# Sidekiq::Client.new(config: config)
|
41
|
+
#
|
42
|
+
# @param pool [ConnectionPool] explicit Redis pool to use
|
43
|
+
# @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
|
44
|
+
# @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
|
45
|
+
def initialize(*args, **kwargs)
|
46
|
+
if args.size == 1 && kwargs.size == 0
|
47
|
+
warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
|
48
|
+
# old calling method, accept 1 pool argument
|
49
|
+
@redis_pool = args[0]
|
50
|
+
@chain = Sidekiq.default_configuration.client_middleware
|
51
|
+
@config = Sidekiq.default_configuration
|
52
|
+
else
|
53
|
+
# new calling method: keyword arguments
|
54
|
+
@config = kwargs[:config] || Sidekiq.default_configuration
|
55
|
+
@redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
|
56
|
+
@chain = kwargs[:chain] || @config&.client_middleware
|
57
|
+
raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
|
58
|
+
end
|
46
59
|
end
|
47
60
|
|
48
61
|
##
|
@@ -135,11 +148,11 @@ module Sidekiq
|
|
135
148
|
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
136
149
|
def self.via(pool)
|
137
150
|
raise ArgumentError, "No pool given" if pool.nil?
|
138
|
-
current_sidekiq_pool = Thread.current[:
|
139
|
-
Thread.current[:
|
151
|
+
current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
|
152
|
+
Thread.current[:sidekiq_redis_pool] = pool
|
140
153
|
yield
|
141
154
|
ensure
|
142
|
-
Thread.current[:
|
155
|
+
Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
|
143
156
|
end
|
144
157
|
|
145
158
|
class << self
|
@@ -201,14 +214,14 @@ module Sidekiq
|
|
201
214
|
conn.pipelined do |pipeline|
|
202
215
|
atomic_push(pipeline, payloads)
|
203
216
|
end
|
204
|
-
rescue
|
217
|
+
rescue RedisClient::Error => ex
|
205
218
|
# 2550 Failover can cause the server to become a replica, need
|
206
219
|
# to disconnect and reopen the socket to get back to the primary.
|
207
220
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
221
|
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
222
|
# The retry logic is copied from sidekiq.rb
|
210
223
|
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
-
conn.
|
224
|
+
conn.close
|
212
225
|
retryable = false
|
213
226
|
retry
|
214
227
|
end
|
data/lib/sidekiq/component.rb
CHANGED
@@ -0,0 +1,270 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
require "set"
|
4
|
+
require "sidekiq/redis_connection"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
# Sidekiq::Config represents the global configuration for an instance of Sidekiq.
|
8
|
+
class Config
|
9
|
+
extend Forwardable
|
10
|
+
|
11
|
+
DEFAULTS = {
|
12
|
+
labels: Set.new,
|
13
|
+
require: ".",
|
14
|
+
environment: nil,
|
15
|
+
concurrency: 5,
|
16
|
+
timeout: 25,
|
17
|
+
poll_interval_average: nil,
|
18
|
+
average_scheduled_poll_interval: 5,
|
19
|
+
on_complex_arguments: :raise,
|
20
|
+
error_handlers: [],
|
21
|
+
death_handlers: [],
|
22
|
+
lifecycle_events: {
|
23
|
+
startup: [],
|
24
|
+
quiet: [],
|
25
|
+
shutdown: [],
|
26
|
+
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
27
|
+
heartbeat: [],
|
28
|
+
# triggers on EVERY heartbeat call, every 10 seconds
|
29
|
+
beat: []
|
30
|
+
},
|
31
|
+
dead_max_jobs: 10_000,
|
32
|
+
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
33
|
+
reloader: proc { |&block| block.call }
|
34
|
+
}
|
35
|
+
|
36
|
+
ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
|
37
|
+
l = cfg.logger
|
38
|
+
l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
|
39
|
+
l.warn("#{ex.class.name}: #{ex.message}")
|
40
|
+
l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
41
|
+
}
|
42
|
+
|
43
|
+
def initialize(options = {})
|
44
|
+
@options = DEFAULTS.merge(options)
|
45
|
+
@options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
|
46
|
+
@directory = {}
|
47
|
+
@redis_config = {}
|
48
|
+
@capsules = {}
|
49
|
+
end
|
50
|
+
|
51
|
+
def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
|
52
|
+
attr_reader :options
|
53
|
+
attr_reader :capsules
|
54
|
+
|
55
|
+
# LEGACY: edits the default capsule
|
56
|
+
# config.concurrency = 5
|
57
|
+
def concurrency=(val)
|
58
|
+
default_capsule.concurrency = Integer(val)
|
59
|
+
end
|
60
|
+
|
61
|
+
def concurrency
|
62
|
+
default_capsule.concurrency
|
63
|
+
end
|
64
|
+
|
65
|
+
def total_concurrency
|
66
|
+
capsules.each_value.sum(&:concurrency)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Edit the default capsule.
|
70
|
+
# config.queues = %w( high default low ) # strict
|
71
|
+
# config.queues = %w( high,3 default,2 low,1 ) # weighted
|
72
|
+
# config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
|
73
|
+
#
|
74
|
+
# With weighted priority, queue will be checked first (weight / total) of the time.
|
75
|
+
# high will be checked first (3/6) or 50% of the time.
|
76
|
+
# I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
|
77
|
+
# are ridiculous and unnecessarily expensive. You can get random queue ordering
|
78
|
+
# by explicitly setting all weights to 1.
|
79
|
+
def queues=(val)
|
80
|
+
default_capsule.queues = val
|
81
|
+
end
|
82
|
+
|
83
|
+
def queues
|
84
|
+
default_capsule.queues
|
85
|
+
end
|
86
|
+
|
87
|
+
def client_middleware
|
88
|
+
@client_chain ||= Sidekiq::Middleware::Chain.new(self)
|
89
|
+
yield @client_chain if block_given?
|
90
|
+
@client_chain
|
91
|
+
end
|
92
|
+
|
93
|
+
def server_middleware
|
94
|
+
@server_chain ||= Sidekiq::Middleware::Chain.new(self)
|
95
|
+
yield @server_chain if block_given?
|
96
|
+
@server_chain
|
97
|
+
end
|
98
|
+
|
99
|
+
def default_capsule(&block)
|
100
|
+
capsule("default", &block)
|
101
|
+
end
|
102
|
+
|
103
|
+
# register a new queue processing subsystem
|
104
|
+
def capsule(name)
|
105
|
+
nm = name.to_s
|
106
|
+
cap = @capsules.fetch(nm) do
|
107
|
+
cap = Sidekiq::Capsule.new(nm, self)
|
108
|
+
@capsules[nm] = cap
|
109
|
+
end
|
110
|
+
yield cap if block_given?
|
111
|
+
cap
|
112
|
+
end
|
113
|
+
|
114
|
+
# All capsules must use the same Redis configuration
|
115
|
+
def redis=(hash)
|
116
|
+
@redis_config = @redis_config.merge(hash)
|
117
|
+
end
|
118
|
+
|
119
|
+
def redis_pool
|
120
|
+
Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
|
121
|
+
end
|
122
|
+
|
123
|
+
private def local_redis_pool
|
124
|
+
# this is our internal client/housekeeping pool. each capsule has its
|
125
|
+
# own pool for executing threads.
|
126
|
+
@redis ||= new_redis_pool(5, "internal")
|
127
|
+
end
|
128
|
+
|
129
|
+
def new_redis_pool(size, name = "unset")
|
130
|
+
# connection pool is lazy, it will not create connections unless you actually need them
|
131
|
+
# so don't be skimpy!
|
132
|
+
RedisConnection.create(@redis_config.merge(size: size, logger: logger, pool_name: name))
|
133
|
+
end
|
134
|
+
|
135
|
+
def redis_info
|
136
|
+
redis do |conn|
|
137
|
+
conn.info
|
138
|
+
rescue RedisClientAdapter::CommandError => ex
|
139
|
+
# 2850 return fake version when INFO command has (probably) been renamed
|
140
|
+
raise unless /unknown command/.match?(ex.message)
|
141
|
+
{
|
142
|
+
"redis_version" => "9.9.9",
|
143
|
+
"uptime_in_days" => "9999",
|
144
|
+
"connected_clients" => "9999",
|
145
|
+
"used_memory_human" => "9P",
|
146
|
+
"used_memory_peak_human" => "9P"
|
147
|
+
}.freeze
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def redis
|
152
|
+
raise ArgumentError, "requires a block" unless block_given?
|
153
|
+
redis_pool.with do |conn|
|
154
|
+
retryable = true
|
155
|
+
begin
|
156
|
+
yield conn
|
157
|
+
rescue RedisClientAdapter::BaseError => ex
|
158
|
+
# 2550 Failover can cause the server to become a replica, need
|
159
|
+
# to disconnect and reopen the socket to get back to the primary.
|
160
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
161
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
162
|
+
# The same retry logic is also used in client.rb
|
163
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
164
|
+
conn.close
|
165
|
+
retryable = false
|
166
|
+
retry
|
167
|
+
end
|
168
|
+
raise
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
# register global singletons which can be accessed elsewhere
|
174
|
+
def register(name, instance)
|
175
|
+
@directory[name] = instance
|
176
|
+
end
|
177
|
+
|
178
|
+
# find a singleton
|
179
|
+
def lookup(name, default_class = nil)
|
180
|
+
# JNDI is just a fancy name for a hash lookup
|
181
|
+
@directory.fetch(name) do |key|
|
182
|
+
return nil unless default_class
|
183
|
+
@directory[key] = default_class.new(self)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
##
|
188
|
+
# Death handlers are called when all retries for a job have been exhausted and
|
189
|
+
# the job dies. It's the notification to your application
|
190
|
+
# that this job will not succeed without manual intervention.
|
191
|
+
#
|
192
|
+
# Sidekiq.configure_server do |config|
|
193
|
+
# config.death_handlers << ->(job, ex) do
|
194
|
+
# end
|
195
|
+
# end
|
196
|
+
def death_handlers
|
197
|
+
@options[:death_handlers]
|
198
|
+
end
|
199
|
+
|
200
|
+
# How frequently Redis should be checked by a random Sidekiq process for
|
201
|
+
# scheduled and retriable jobs. Each individual process will take turns by
|
202
|
+
# waiting some multiple of this value.
|
203
|
+
#
|
204
|
+
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
205
|
+
def average_scheduled_poll_interval=(interval)
|
206
|
+
@options[:average_scheduled_poll_interval] = interval
|
207
|
+
end
|
208
|
+
|
209
|
+
# Register a proc to handle any error which occurs within the Sidekiq process.
|
210
|
+
#
|
211
|
+
# Sidekiq.configure_server do |config|
|
212
|
+
# config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
|
213
|
+
# end
|
214
|
+
#
|
215
|
+
# The default error handler logs errors to @logger.
|
216
|
+
def error_handlers
|
217
|
+
@options[:error_handlers]
|
218
|
+
end
|
219
|
+
|
220
|
+
# Register a block to run at a point in the Sidekiq lifecycle.
|
221
|
+
# :startup, :quiet or :shutdown are valid events.
|
222
|
+
#
|
223
|
+
# Sidekiq.configure_server do |config|
|
224
|
+
# config.on(:shutdown) do
|
225
|
+
# puts "Goodbye cruel world!"
|
226
|
+
# end
|
227
|
+
# end
|
228
|
+
def on(event, &block)
|
229
|
+
raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
|
230
|
+
raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
|
231
|
+
@options[:lifecycle_events][event] << block
|
232
|
+
end
|
233
|
+
|
234
|
+
def logger
|
235
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
|
236
|
+
log.level = Logger::INFO
|
237
|
+
log.formatter = if ENV["DYNO"]
|
238
|
+
Sidekiq::Logger::Formatters::WithoutTimestamp.new
|
239
|
+
else
|
240
|
+
Sidekiq::Logger::Formatters::Pretty.new
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def logger=(logger)
|
246
|
+
if logger.nil?
|
247
|
+
self.logger.level = Logger::FATAL
|
248
|
+
return
|
249
|
+
end
|
250
|
+
|
251
|
+
logger.extend(Sidekiq::LoggingUtils)
|
252
|
+
@logger = logger
|
253
|
+
end
|
254
|
+
|
255
|
+
# INTERNAL USE ONLY
|
256
|
+
def handle_exception(ex, ctx = {})
|
257
|
+
if @options[:error_handlers].size == 0
|
258
|
+
p ["!!!!!", ex]
|
259
|
+
end
|
260
|
+
@options[:error_handlers].each do |handler|
|
261
|
+
handler.call(ex, ctx, self)
|
262
|
+
rescue => e
|
263
|
+
l = logger
|
264
|
+
l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
265
|
+
l.error e
|
266
|
+
l.error e.backtrace.join("\n") unless e.backtrace.nil?
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require "sidekiq/redis_connection"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
# This file is designed to be required within the user's
|
5
|
+
# deployment script; it should need a bare minimum of dependencies.
|
6
|
+
# Usage:
|
7
|
+
#
|
8
|
+
# require "sidekiq/deploy"
|
9
|
+
# Sidekiq::Deploy.mark!("Some change")
|
10
|
+
#
|
11
|
+
# If you do not pass a label, Sidekiq will try to use the latest
|
12
|
+
# git commit info.
|
13
|
+
#
|
14
|
+
|
15
|
+
module Sidekiq
|
16
|
+
class Deploy
|
17
|
+
MARK_TTL = 90 * 24 * 60 * 60 # 90 days
|
18
|
+
|
19
|
+
LABEL_MAKER = -> {
|
20
|
+
`git log -1 --format="%h %s"`.strip
|
21
|
+
}
|
22
|
+
|
23
|
+
def self.mark!(label = nil)
|
24
|
+
label ||= LABEL_MAKER.call
|
25
|
+
Sidekiq::Deploy.new.mark(label: label)
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize(pool = Sidekiq::RedisConnection.create)
|
29
|
+
@pool = pool
|
30
|
+
end
|
31
|
+
|
32
|
+
def mark(at: Time.now, label: "")
|
33
|
+
# we need to round the timestamp so that we gracefully
|
34
|
+
# handle an very common error in marking deploys:
|
35
|
+
# having every process mark its deploy, leading
|
36
|
+
# to N marks for each deploy. Instead we round the time
|
37
|
+
# to the minute so that multple marks within that minute
|
38
|
+
# will all naturally rollup into one mark per minute.
|
39
|
+
whence = at.utc
|
40
|
+
floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
|
41
|
+
datecode = floor.strftime("%Y%m%d")
|
42
|
+
key = "#{datecode}-marks"
|
43
|
+
stamp = floor.iso8601
|
44
|
+
|
45
|
+
@pool.with do |c|
|
46
|
+
# only allow one deploy mark for a given label for the next minute
|
47
|
+
lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
|
48
|
+
if lock
|
49
|
+
c.multi do |pipe|
|
50
|
+
pipe.hsetnx(key, stamp, label)
|
51
|
+
pipe.expire(key, MARK_TTL)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def fetch(date = Time.now.utc.to_date)
|
58
|
+
datecode = date.strftime("%Y%m%d")
|
59
|
+
@pool.with { |c| c.hgetall("#{datecode}-marks") }
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require "sidekiq/component"
|
2
|
+
require "sidekiq/launcher"
|
3
|
+
require "sidekiq/metrics/tracking"
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
class Embedded
|
7
|
+
include Sidekiq::Component
|
8
|
+
|
9
|
+
def initialize(config)
|
10
|
+
@config = config
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
housekeeping
|
15
|
+
fire_event(:startup, reverse: false, reraise: true)
|
16
|
+
@launcher = Sidekiq::Launcher.new(@config, embedded: true)
|
17
|
+
@launcher.run
|
18
|
+
sleep 0.1 # pause to give threads time to spin up
|
19
|
+
|
20
|
+
logger.info "Embedded mode running with #{Thread.list.size} threads"
|
21
|
+
logger.debug { Thread.list.map(&:name) }
|
22
|
+
end
|
23
|
+
|
24
|
+
def quiet
|
25
|
+
@launcher&.quiet
|
26
|
+
end
|
27
|
+
|
28
|
+
def stop
|
29
|
+
@launcher&.stop
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def housekeeping
|
35
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
36
|
+
logger.info Sidekiq::LICENSE
|
37
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
38
|
+
|
39
|
+
# touch the connection pool so it is created before we
|
40
|
+
# fire startup and start multithreading.
|
41
|
+
info = config.redis_info
|
42
|
+
ver = Gem::Version.new(info["redis_version"])
|
43
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
44
|
+
|
45
|
+
maxmemory_policy = info["maxmemory_policy"]
|
46
|
+
if maxmemory_policy != "noeviction"
|
47
|
+
logger.warn <<~EOM
|
48
|
+
|
49
|
+
|
50
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
51
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
52
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
53
|
+
|
54
|
+
EOM
|
55
|
+
end
|
56
|
+
|
57
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
58
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|