sidekiq 0.10.0 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/Changes.md +2082 -0
- data/LICENSE.txt +9 -0
- data/README.md +73 -27
- data/bin/sidekiq +25 -9
- data/bin/sidekiqload +247 -0
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/job.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +1145 -0
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +348 -109
- data/lib/sidekiq/client.rb +241 -41
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +88 -0
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +301 -0
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +271 -0
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +96 -103
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +149 -38
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +55 -0
- data/lib/sidekiq/processor.rb +246 -61
- data/lib/sidekiq/rails.rb +60 -13
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +68 -15
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +236 -0
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +30 -0
- data/lib/sidekiq/testing.rb +310 -10
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +4 -1
- data/lib/sidekiq/web/action.rb +93 -0
- data/lib/sidekiq/web/application.rb +463 -0
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +364 -0
- data/lib/sidekiq/web/router.rb +104 -0
- data/lib/sidekiq/web.rb +143 -74
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +120 -73
- data/sidekiq.gemspec +26 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status.png +0 -0
- data/web/assets/javascripts/application.js +177 -3
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +57 -0
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +153 -0
- data/web/assets/stylesheets/application.css +729 -7
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +5 -0
- data/web/locales/ar.yml +87 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +75 -0
- data/web/locales/de.yml +81 -0
- data/web/locales/el.yml +87 -0
- data/web/locales/en.yml +101 -0
- data/web/locales/es.yml +86 -0
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +99 -0
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +80 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +91 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +96 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +83 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +77 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +95 -0
- data/web/locales/zh-tw.yml +102 -0
- data/web/views/_footer.erb +23 -0
- data/web/views/_job_info.erb +105 -0
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +52 -0
- data/web/views/_paging.erb +25 -0
- data/web/views/_poll_link.erb +4 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +148 -0
- data/web/views/dashboard.erb +105 -0
- data/web/views/dead.erb +34 -0
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +42 -0
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +74 -0
- data/web/views/queue.erb +55 -0
- data/web/views/queues.erb +44 -0
- data/web/views/retries.erb +79 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +56 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +159 -237
- data/.gitignore +0 -6
- data/.rvmrc +0 -4
- data/COMM-LICENSE +0 -75
- data/Gemfile +0 -10
- data/LICENSE +0 -22
- data/Rakefile +0 -9
- data/TODO.md +0 -1
- data/bin/client +0 -7
- data/bin/sidekiqctl +0 -43
- data/config.ru +0 -8
- data/examples/chef/cookbooks/sidekiq/README.rdoc +0 -11
- data/examples/chef/cookbooks/sidekiq/recipes/default.rb +0 -55
- data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb +0 -8
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb +0 -219
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb +0 -22
- data/examples/config.yml +0 -9
- data/examples/monitrc.conf +0 -6
- data/examples/por.rb +0 -27
- data/examples/scheduling.rb +0 -37
- data/examples/sinkiq.rb +0 -57
- data/examples/web-ui.png +0 -0
- data/lib/sidekiq/capistrano.rb +0 -32
- data/lib/sidekiq/extensions/action_mailer.rb +0 -26
- data/lib/sidekiq/extensions/active_record.rb +0 -27
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -21
- data/lib/sidekiq/middleware/client/unique_jobs.rb +0 -32
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/exception_handler.rb +0 -38
- data/lib/sidekiq/middleware/server/failure_jobs.rb +0 -24
- data/lib/sidekiq/middleware/server/logging.rb +0 -27
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -59
- data/lib/sidekiq/middleware/server/unique_jobs.rb +0 -15
- data/lib/sidekiq/retry.rb +0 -57
- data/lib/sidekiq/util.rb +0 -61
- data/lib/sidekiq/worker.rb +0 -37
- data/myapp/.gitignore +0 -15
- data/myapp/Capfile +0 -5
- data/myapp/Gemfile +0 -19
- data/myapp/Gemfile.lock +0 -143
- data/myapp/Rakefile +0 -7
- data/myapp/app/controllers/application_controller.rb +0 -3
- data/myapp/app/controllers/work_controller.rb +0 -38
- data/myapp/app/helpers/application_helper.rb +0 -2
- data/myapp/app/mailers/.gitkeep +0 -0
- data/myapp/app/mailers/user_mailer.rb +0 -9
- data/myapp/app/models/.gitkeep +0 -0
- data/myapp/app/models/post.rb +0 -5
- data/myapp/app/views/layouts/application.html.erb +0 -14
- data/myapp/app/views/user_mailer/greetings.html.erb +0 -3
- data/myapp/app/views/work/index.html.erb +0 -1
- data/myapp/app/workers/hard_worker.rb +0 -9
- data/myapp/config/application.rb +0 -59
- data/myapp/config/boot.rb +0 -6
- data/myapp/config/database.yml +0 -25
- data/myapp/config/deploy.rb +0 -15
- data/myapp/config/environment.rb +0 -5
- data/myapp/config/environments/development.rb +0 -38
- data/myapp/config/environments/production.rb +0 -67
- data/myapp/config/environments/test.rb +0 -37
- data/myapp/config/initializers/backtrace_silencers.rb +0 -7
- data/myapp/config/initializers/inflections.rb +0 -15
- data/myapp/config/initializers/mime_types.rb +0 -5
- data/myapp/config/initializers/secret_token.rb +0 -7
- data/myapp/config/initializers/session_store.rb +0 -8
- data/myapp/config/initializers/sidekiq.rb +0 -6
- data/myapp/config/initializers/wrap_parameters.rb +0 -14
- data/myapp/config/locales/en.yml +0 -5
- data/myapp/config/routes.rb +0 -10
- data/myapp/config.ru +0 -4
- data/myapp/db/migrate/20120123214055_create_posts.rb +0 -10
- data/myapp/db/seeds.rb +0 -7
- data/myapp/lib/assets/.gitkeep +0 -0
- data/myapp/lib/tasks/.gitkeep +0 -0
- data/myapp/log/.gitkeep +0 -0
- data/myapp/script/rails +0 -6
- data/test/config.yml +0 -9
- data/test/fake_env.rb +0 -0
- data/test/helper.rb +0 -15
- data/test/test_cli.rb +0 -168
- data/test/test_client.rb +0 -105
- data/test/test_extensions.rb +0 -68
- data/test/test_manager.rb +0 -43
- data/test/test_middleware.rb +0 -92
- data/test/test_processor.rb +0 -32
- data/test/test_retry.rb +0 -83
- data/test/test_stats.rb +0 -78
- data/test/test_testing.rb +0 -65
- data/test/test_web.rb +0 -61
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +0 -91
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +0 -98
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +0 -154
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +0 -136
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +0 -92
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +0 -210
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +0 -95
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +0 -125
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +0 -130
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +0 -270
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +0 -51
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +0 -271
- data/web/assets/javascripts/vendor/bootstrap.js +0 -12
- data/web/assets/javascripts/vendor/jquery.js +0 -9266
- data/web/assets/stylesheets/vendor/bootstrap-responsive.css +0 -567
- data/web/assets/stylesheets/vendor/bootstrap.css +0 -3365
- data/web/views/index.slim +0 -62
- data/web/views/layout.slim +0 -24
- data/web/views/queue.slim +0 -11
@@ -0,0 +1,127 @@
|
|
1
|
+
require "sidekiq/component"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# A Sidekiq::Capsule is the set of resources necessary to
|
5
|
+
# process one or more queues with a given concurrency.
|
6
|
+
# One "default" Capsule is started but the user may declare additional
|
7
|
+
# Capsules in their initializer.
|
8
|
+
#
|
9
|
+
# This capsule will pull jobs from the "single" queue and process
|
10
|
+
# the jobs with one thread, meaning the jobs will be processed serially.
|
11
|
+
#
|
12
|
+
# Sidekiq.configure_server do |config|
|
13
|
+
# config.capsule("single-threaded") do |cap|
|
14
|
+
# cap.concurrency = 1
|
15
|
+
# cap.queues = %w(single)
|
16
|
+
# end
|
17
|
+
# end
|
18
|
+
class Capsule
|
19
|
+
include Sidekiq::Component
|
20
|
+
|
21
|
+
attr_reader :name
|
22
|
+
attr_reader :queues
|
23
|
+
attr_accessor :concurrency
|
24
|
+
attr_reader :mode
|
25
|
+
attr_reader :weights
|
26
|
+
|
27
|
+
def initialize(name, config)
|
28
|
+
@name = name
|
29
|
+
@config = config
|
30
|
+
@queues = ["default"]
|
31
|
+
@weights = {"default" => 0}
|
32
|
+
@concurrency = config[:concurrency]
|
33
|
+
@mode = :strict
|
34
|
+
end
|
35
|
+
|
36
|
+
def fetcher
|
37
|
+
@fetcher ||= begin
|
38
|
+
inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
|
39
|
+
inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
|
40
|
+
inst
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def stop
|
45
|
+
fetcher&.bulk_requeue([])
|
46
|
+
end
|
47
|
+
|
48
|
+
# Sidekiq checks queues in three modes:
|
49
|
+
# - :strict - all queues have 0 weight and are checked strictly in order
|
50
|
+
# - :weighted - queues have arbitrary weight between 1 and N
|
51
|
+
# - :random - all queues have weight of 1
|
52
|
+
def queues=(val)
|
53
|
+
@weights = {}
|
54
|
+
@queues = Array(val).each_with_object([]) do |qstr, memo|
|
55
|
+
arr = qstr
|
56
|
+
arr = qstr.split(",") if qstr.is_a?(String)
|
57
|
+
name, weight = arr
|
58
|
+
@weights[name] = weight.to_i
|
59
|
+
[weight.to_i, 1].max.times do
|
60
|
+
memo << name
|
61
|
+
end
|
62
|
+
end
|
63
|
+
@mode = if @weights.values.all?(&:zero?)
|
64
|
+
:strict
|
65
|
+
elsif @weights.values.all? { |x| x == 1 }
|
66
|
+
:random
|
67
|
+
else
|
68
|
+
:weighted
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# Allow the middleware to be different per-capsule.
|
73
|
+
# Avoid if possible and add middleware globally so all
|
74
|
+
# capsules share the same chains. Easier to debug that way.
|
75
|
+
def client_middleware
|
76
|
+
@client_chain ||= config.client_middleware.copy_for(self)
|
77
|
+
yield @client_chain if block_given?
|
78
|
+
@client_chain
|
79
|
+
end
|
80
|
+
|
81
|
+
def server_middleware
|
82
|
+
@server_chain ||= config.server_middleware.copy_for(self)
|
83
|
+
yield @server_chain if block_given?
|
84
|
+
@server_chain
|
85
|
+
end
|
86
|
+
|
87
|
+
def redis_pool
|
88
|
+
Thread.current[:sidekiq_redis_pool] || local_redis_pool
|
89
|
+
end
|
90
|
+
|
91
|
+
def local_redis_pool
|
92
|
+
# connection pool is lazy, it will not create connections unless you actually need them
|
93
|
+
# so don't be skimpy!
|
94
|
+
@redis ||= config.new_redis_pool(@concurrency, name)
|
95
|
+
end
|
96
|
+
|
97
|
+
def redis
|
98
|
+
raise ArgumentError, "requires a block" unless block_given?
|
99
|
+
redis_pool.with do |conn|
|
100
|
+
retryable = true
|
101
|
+
begin
|
102
|
+
yield conn
|
103
|
+
rescue RedisClientAdapter::BaseError => ex
|
104
|
+
# 2550 Failover can cause the server to become a replica, need
|
105
|
+
# to disconnect and reopen the socket to get back to the primary.
|
106
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
107
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
108
|
+
# The same retry logic is also used in client.rb
|
109
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
110
|
+
conn.close
|
111
|
+
retryable = false
|
112
|
+
retry
|
113
|
+
end
|
114
|
+
raise
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def lookup(name)
|
120
|
+
config.lookup(name)
|
121
|
+
end
|
122
|
+
|
123
|
+
def logger
|
124
|
+
config.logger
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -1,184 +1,423 @@
|
|
1
|
-
|
2
|
-
# Handle Ctrl-C in JRuby like MRI
|
3
|
-
# http://jira.codehaus.org/browse/JRUBY-4637
|
4
|
-
Thread.main.raise Interrupt
|
5
|
-
end
|
1
|
+
# frozen_string_literal: true
|
6
2
|
|
7
|
-
|
8
|
-
# Heroku sends TERM and then waits 10 seconds for process to exit.
|
9
|
-
Thread.main.raise Interrupt
|
10
|
-
end
|
3
|
+
$stdout.sync = true
|
11
4
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
5
|
+
require "yaml"
|
6
|
+
require "singleton"
|
7
|
+
require "optparse"
|
8
|
+
require "erb"
|
9
|
+
require "fileutils"
|
17
10
|
|
18
|
-
require
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require 'sidekiq/manager'
|
11
|
+
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
13
|
+
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
15
|
+
require "sidekiq/launcher"
|
24
16
|
|
25
|
-
module Sidekiq
|
17
|
+
module Sidekiq # :nodoc:
|
26
18
|
class CLI
|
27
|
-
include
|
28
|
-
include Singleton
|
19
|
+
include Sidekiq::Component
|
20
|
+
include Singleton unless $TESTING
|
29
21
|
|
30
|
-
|
31
|
-
attr_accessor :
|
32
|
-
attr_accessor :
|
22
|
+
attr_accessor :launcher
|
23
|
+
attr_accessor :environment
|
24
|
+
attr_accessor :config
|
25
|
+
|
26
|
+
def parse(args = ARGV.dup)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
28
|
+
|
29
|
+
setup_options(args)
|
30
|
+
initialize_logger
|
31
|
+
validate!
|
32
|
+
end
|
33
33
|
|
34
|
-
def
|
35
|
-
|
34
|
+
def jruby?
|
35
|
+
defined?(::JRUBY_VERSION)
|
36
36
|
end
|
37
37
|
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
# Code within this method is not tested because it alters
|
39
|
+
# global process state irreversibly. PRs which improve the
|
40
|
+
# test coverage of Sidekiq::CLI are welcomed.
|
41
|
+
def run(boot_app: true)
|
42
|
+
boot_application if boot_app
|
43
|
+
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
45
|
+
print_banner
|
46
|
+
end
|
47
|
+
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
48
|
+
|
49
|
+
self_read, self_write = IO.pipe
|
50
|
+
sigs = %w[INT TERM TTIN TSTP]
|
51
|
+
# USR1 and USR2 don't work on the JVM
|
52
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
53
|
+
sigs.each do |sig|
|
54
|
+
old_handler = Signal.trap(sig) do
|
55
|
+
if old_handler.respond_to?(:call)
|
56
|
+
begin
|
57
|
+
old_handler.call
|
58
|
+
rescue Exception => exc
|
59
|
+
# signal handlers can't use Logger so puts only
|
60
|
+
puts ["Error in #{sig} handler", exc].inspect
|
61
|
+
end
|
62
|
+
end
|
63
|
+
self_write.puts(sig)
|
64
|
+
end
|
65
|
+
rescue ArgumentError
|
66
|
+
puts "Signal #{sig} not supported"
|
67
|
+
end
|
41
68
|
|
42
|
-
|
43
|
-
|
44
|
-
|
69
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
70
|
+
logger.info Sidekiq::LICENSE
|
71
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
45
72
|
|
46
|
-
|
47
|
-
|
73
|
+
# touch the connection pool so it is created before we
|
74
|
+
# fire startup and start multithreading.
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
48
78
|
|
49
|
-
|
50
|
-
|
51
|
-
|
79
|
+
maxmemory_policy = info["maxmemory_policy"]
|
80
|
+
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
81
|
+
# Redis Enterprise Cloud returns "" for their policy 😳
|
82
|
+
logger.warn <<~EOM
|
83
|
+
|
84
|
+
|
85
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
86
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
87
|
+
See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
|
88
|
+
|
89
|
+
EOM
|
90
|
+
end
|
91
|
+
|
92
|
+
# Since the user can pass us a connection pool explicitly in the initializer, we
|
93
|
+
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
94
|
+
@config.capsules.each_pair do |name, cap|
|
95
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
96
|
+
end
|
97
|
+
|
98
|
+
# cache process identity
|
99
|
+
@config[:identity] = identity
|
100
|
+
|
101
|
+
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
102
|
+
@config.server_middleware
|
103
|
+
|
104
|
+
# Before this point, the process is initializing with just the main thread.
|
105
|
+
# Starting here the process will now have multiple threads running.
|
106
|
+
fire_event(:startup, reverse: false, reraise: true)
|
107
|
+
|
108
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
109
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
110
|
+
|
111
|
+
launch(self_read)
|
52
112
|
end
|
53
113
|
|
54
|
-
def
|
55
|
-
|
56
|
-
|
114
|
+
def launch(self_read)
|
115
|
+
if environment == "development" && $stdout.tty?
|
116
|
+
logger.info "Starting processing, hit Ctrl-C to stop"
|
117
|
+
end
|
118
|
+
|
119
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
120
|
+
|
57
121
|
begin
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
122
|
+
launcher.run
|
123
|
+
|
124
|
+
while self_read.wait_readable
|
125
|
+
signal = self_read.gets.strip
|
126
|
+
handle_signal(signal)
|
127
|
+
end
|
62
128
|
rescue Interrupt
|
63
|
-
logger.info
|
64
|
-
|
65
|
-
|
66
|
-
|
129
|
+
logger.info "Shutting down"
|
130
|
+
launcher.stop
|
131
|
+
logger.info "Bye!"
|
132
|
+
|
133
|
+
# Explicitly exit so busy Processor threads won't block process shutdown.
|
134
|
+
#
|
135
|
+
# NB: slow at_exit handlers will prevent a timely exit if they take
|
136
|
+
# a while to run. If Sidekiq is getting here but the process isn't exiting,
|
137
|
+
# use the TTIN signal to determine where things are stuck.
|
138
|
+
exit(0)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
HOLIDAY_COLORS = {
|
143
|
+
# got other color-specific holidays from around the world?
|
144
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
145
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
146
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
147
|
+
}
|
148
|
+
|
149
|
+
def self.day
|
150
|
+
@@day ||= begin
|
151
|
+
t = Date.today
|
152
|
+
"#{t.month}-#{t.day}"
|
67
153
|
end
|
68
154
|
end
|
69
155
|
|
156
|
+
def self.r
|
157
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.b
|
161
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.w
|
165
|
+
"\e[1;37m"
|
166
|
+
end
|
167
|
+
|
168
|
+
def self.reset
|
169
|
+
@@b = @@r = @@day = nil
|
170
|
+
"\e[0m"
|
171
|
+
end
|
172
|
+
|
173
|
+
def self.banner
|
174
|
+
%{
|
175
|
+
#{w} m,
|
176
|
+
#{w} `$b
|
177
|
+
#{w} .ss, $$: .,d$
|
178
|
+
#{w} `$$P,d$P' .,md$P"'
|
179
|
+
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
180
|
+
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
181
|
+
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
182
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
183
|
+
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
184
|
+
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
185
|
+
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
186
|
+
#{w} .d$$ #{r} |_|
|
187
|
+
#{reset}}
|
188
|
+
end
|
189
|
+
|
190
|
+
SIGNAL_HANDLERS = {
|
191
|
+
# Ctrl-C in terminal
|
192
|
+
"INT" => ->(cli) { raise Interrupt },
|
193
|
+
# TERM is the signal that Sidekiq must exit.
|
194
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
195
|
+
"TERM" => ->(cli) { raise Interrupt },
|
196
|
+
"TSTP" => ->(cli) {
|
197
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
198
|
+
cli.launcher.quiet
|
199
|
+
},
|
200
|
+
"TTIN" => ->(cli) {
|
201
|
+
Thread.list.each do |thread|
|
202
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
203
|
+
if thread.backtrace
|
204
|
+
cli.logger.warn thread.backtrace.join("\n")
|
205
|
+
else
|
206
|
+
cli.logger.warn "<no backtrace available>"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
}
|
210
|
+
}
|
211
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
212
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
213
|
+
|
214
|
+
def handle_signal(sig)
|
215
|
+
logger.debug "Got #{sig} signal"
|
216
|
+
SIGNAL_HANDLERS[sig].call(self)
|
217
|
+
end
|
218
|
+
|
70
219
|
private
|
71
220
|
|
72
|
-
def
|
73
|
-
|
221
|
+
def print_banner
|
222
|
+
puts "\e[31m"
|
223
|
+
puts Sidekiq::CLI.banner
|
224
|
+
puts "\e[0m"
|
74
225
|
end
|
75
226
|
|
76
|
-
def
|
77
|
-
|
227
|
+
def set_environment(cli_env)
|
228
|
+
# See #984 for discussion.
|
229
|
+
# APP_ENV is now the preferred ENV term since it is not tech-specific.
|
230
|
+
# Both Sinatra 2.0+ and Sidekiq support this term.
|
231
|
+
# RAILS_ENV and RACK_ENV are there for legacy support.
|
232
|
+
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
233
|
+
config[:environment] = @environment
|
78
234
|
end
|
79
235
|
|
80
|
-
def
|
81
|
-
|
236
|
+
def symbolize_keys_deep!(hash)
|
237
|
+
hash.keys.each do |k|
|
238
|
+
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
239
|
+
hash[symkey] = hash.delete k
|
240
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
241
|
+
end
|
82
242
|
end
|
83
243
|
|
84
|
-
|
85
|
-
|
244
|
+
alias_method :die, :exit
|
245
|
+
alias_method :☠, :exit
|
86
246
|
|
87
|
-
|
247
|
+
def setup_options(args)
|
248
|
+
# parse CLI options
|
249
|
+
opts = parse_options(args)
|
88
250
|
|
89
|
-
|
90
|
-
|
91
|
-
|
251
|
+
set_environment opts[:environment]
|
252
|
+
|
253
|
+
# check config file presence
|
254
|
+
if opts[:config_file]
|
255
|
+
unless File.exist?(opts[:config_file])
|
256
|
+
raise ArgumentError, "No such file #{opts[:config_file]}"
|
257
|
+
end
|
92
258
|
else
|
93
|
-
|
259
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
260
|
+
File.join(opts[:require], "config")
|
261
|
+
else
|
262
|
+
File.join(@config[:require], "config")
|
263
|
+
end
|
264
|
+
|
265
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
266
|
+
path = File.join(config_dir, config_file)
|
267
|
+
opts[:config_file] ||= path if File.exist?(path)
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
# parse config file options
|
272
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
273
|
+
|
274
|
+
# set defaults
|
275
|
+
opts[:queues] = ["default"] if opts[:queues].nil?
|
276
|
+
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
277
|
+
|
278
|
+
# merge with defaults
|
279
|
+
@config.merge!(opts)
|
280
|
+
|
281
|
+
@config.default_capsule.tap do |cap|
|
282
|
+
cap.queues = opts[:queues]
|
283
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
284
|
+
end
|
285
|
+
|
286
|
+
opts[:capsules]&.each do |name, cap_config|
|
287
|
+
@config.capsule(name.to_s) do |cap|
|
288
|
+
cap.queues = cap_config[:queues]
|
289
|
+
cap.concurrency = cap_config[:concurrency]
|
290
|
+
end
|
94
291
|
end
|
95
292
|
end
|
96
293
|
|
97
|
-
def
|
98
|
-
|
99
|
-
options[:queues].shuffle!
|
294
|
+
def boot_application
|
295
|
+
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
100
296
|
|
101
|
-
if
|
102
|
-
|
297
|
+
if File.directory?(@config[:require])
|
298
|
+
require "rails"
|
299
|
+
if ::Rails::VERSION::MAJOR < 6
|
300
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
301
|
+
end
|
302
|
+
require "sidekiq/rails"
|
303
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
304
|
+
@config[:tag] ||= default_tag
|
305
|
+
else
|
306
|
+
require @config[:require]
|
307
|
+
end
|
308
|
+
end
|
309
|
+
|
310
|
+
def default_tag
|
311
|
+
dir = ::Rails.root
|
312
|
+
name = File.basename(dir)
|
313
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
314
|
+
if name.to_i != 0 && prevdir
|
315
|
+
if File.basename(prevdir) == "releases"
|
316
|
+
return File.basename(File.dirname(prevdir))
|
317
|
+
end
|
318
|
+
end
|
319
|
+
name
|
320
|
+
end
|
321
|
+
|
322
|
+
def validate!
|
323
|
+
if !File.exist?(@config[:require]) ||
|
324
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
103
325
|
logger.info "=================================================================="
|
104
|
-
logger.info " Please point
|
105
|
-
logger.info " to load your
|
326
|
+
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
327
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
106
328
|
logger.info "=================================================================="
|
107
329
|
logger.info @parser
|
108
330
|
die(1)
|
109
331
|
end
|
332
|
+
|
333
|
+
[:concurrency, :timeout].each do |opt|
|
334
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
335
|
+
end
|
110
336
|
end
|
111
337
|
|
112
338
|
def parse_options(argv)
|
113
339
|
opts = {}
|
340
|
+
@parser = option_parser(opts)
|
341
|
+
@parser.parse!(argv)
|
342
|
+
opts
|
343
|
+
end
|
114
344
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
345
|
+
def option_parser(opts)
|
346
|
+
parser = OptionParser.new { |o|
|
347
|
+
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
348
|
+
opts[:concurrency] = Integer(arg)
|
119
349
|
end
|
120
350
|
|
121
|
-
o.on "-
|
122
|
-
|
351
|
+
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
352
|
+
opts[:environment] = arg
|
123
353
|
end
|
124
354
|
|
125
|
-
o.on
|
126
|
-
opts[:
|
355
|
+
o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
|
356
|
+
opts[:tag] = arg
|
127
357
|
end
|
128
358
|
|
129
|
-
o.on
|
130
|
-
opts[:
|
359
|
+
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
360
|
+
opts[:queues] ||= []
|
361
|
+
opts[:queues] << arg
|
131
362
|
end
|
132
363
|
|
133
|
-
o.on
|
364
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
134
365
|
opts[:require] = arg
|
135
366
|
end
|
136
367
|
|
137
|
-
o.on
|
138
|
-
opts[:
|
368
|
+
o.on "-t", "--timeout NUM", "Shutdown timeout" do |arg|
|
369
|
+
opts[:timeout] = Integer(arg)
|
139
370
|
end
|
140
371
|
|
141
|
-
o.on
|
142
|
-
opts[:
|
372
|
+
o.on "-v", "--verbose", "Print more verbose output" do |arg|
|
373
|
+
opts[:verbose] = arg
|
143
374
|
end
|
144
375
|
|
145
|
-
o.on
|
376
|
+
o.on "-C", "--config PATH", "path to YAML config file" do |arg|
|
146
377
|
opts[:config_file] = arg
|
147
378
|
end
|
148
|
-
end
|
149
379
|
|
150
|
-
|
151
|
-
|
152
|
-
|
380
|
+
o.on "-V", "--version", "Print version and exit" do
|
381
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
382
|
+
die(0)
|
383
|
+
end
|
384
|
+
}
|
385
|
+
|
386
|
+
parser.banner = "sidekiq [options]"
|
387
|
+
parser.on_tail "-h", "--help", "Show help" do
|
388
|
+
logger.info parser
|
153
389
|
die 1
|
154
390
|
end
|
155
|
-
|
156
|
-
|
391
|
+
|
392
|
+
parser
|
157
393
|
end
|
158
394
|
|
159
|
-
def
|
160
|
-
|
161
|
-
File.open(path, 'w') do |f|
|
162
|
-
f.puts Process.pid
|
163
|
-
end
|
164
|
-
end
|
395
|
+
def initialize_logger
|
396
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
165
397
|
end
|
166
398
|
|
167
|
-
def parse_config(
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
399
|
+
def parse_config(path)
|
400
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
401
|
+
erb.filename = File.expand_path(path)
|
402
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
403
|
+
|
404
|
+
if opts.respond_to? :deep_symbolize_keys!
|
405
|
+
opts.deep_symbolize_keys!
|
406
|
+
else
|
407
|
+
symbolize_keys_deep!(opts)
|
173
408
|
end
|
409
|
+
|
410
|
+
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
411
|
+
opts.delete(:strict)
|
412
|
+
|
174
413
|
opts
|
175
414
|
end
|
176
415
|
|
177
|
-
def
|
178
|
-
(
|
179
|
-
(opts[:queues] ||= []) << q
|
180
|
-
end
|
416
|
+
def rails_app?
|
417
|
+
defined?(::Rails) && ::Rails.respond_to?(:application)
|
181
418
|
end
|
182
|
-
|
183
419
|
end
|
184
420
|
end
|
421
|
+
|
422
|
+
require "sidekiq/systemd"
|
423
|
+
require "sidekiq/metrics/tracking"
|