sidekiq 0.10.0 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/Changes.md +2082 -0
- data/LICENSE.txt +9 -0
- data/README.md +73 -27
- data/bin/sidekiq +25 -9
- data/bin/sidekiqload +247 -0
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/job.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +1145 -0
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +348 -109
- data/lib/sidekiq/client.rb +241 -41
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +88 -0
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +301 -0
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +271 -0
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +96 -103
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +149 -38
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +55 -0
- data/lib/sidekiq/processor.rb +246 -61
- data/lib/sidekiq/rails.rb +60 -13
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +68 -15
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +236 -0
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +30 -0
- data/lib/sidekiq/testing.rb +310 -10
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +4 -1
- data/lib/sidekiq/web/action.rb +93 -0
- data/lib/sidekiq/web/application.rb +463 -0
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +364 -0
- data/lib/sidekiq/web/router.rb +104 -0
- data/lib/sidekiq/web.rb +143 -74
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +120 -73
- data/sidekiq.gemspec +26 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status.png +0 -0
- data/web/assets/javascripts/application.js +177 -3
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +57 -0
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +153 -0
- data/web/assets/stylesheets/application.css +729 -7
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +5 -0
- data/web/locales/ar.yml +87 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +75 -0
- data/web/locales/de.yml +81 -0
- data/web/locales/el.yml +87 -0
- data/web/locales/en.yml +101 -0
- data/web/locales/es.yml +86 -0
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +99 -0
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +80 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +91 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +96 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +83 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +77 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +95 -0
- data/web/locales/zh-tw.yml +102 -0
- data/web/views/_footer.erb +23 -0
- data/web/views/_job_info.erb +105 -0
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +52 -0
- data/web/views/_paging.erb +25 -0
- data/web/views/_poll_link.erb +4 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +148 -0
- data/web/views/dashboard.erb +105 -0
- data/web/views/dead.erb +34 -0
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +42 -0
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +74 -0
- data/web/views/queue.erb +55 -0
- data/web/views/queues.erb +44 -0
- data/web/views/retries.erb +79 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +56 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +159 -237
- data/.gitignore +0 -6
- data/.rvmrc +0 -4
- data/COMM-LICENSE +0 -75
- data/Gemfile +0 -10
- data/LICENSE +0 -22
- data/Rakefile +0 -9
- data/TODO.md +0 -1
- data/bin/client +0 -7
- data/bin/sidekiqctl +0 -43
- data/config.ru +0 -8
- data/examples/chef/cookbooks/sidekiq/README.rdoc +0 -11
- data/examples/chef/cookbooks/sidekiq/recipes/default.rb +0 -55
- data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb +0 -8
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb +0 -219
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb +0 -22
- data/examples/config.yml +0 -9
- data/examples/monitrc.conf +0 -6
- data/examples/por.rb +0 -27
- data/examples/scheduling.rb +0 -37
- data/examples/sinkiq.rb +0 -57
- data/examples/web-ui.png +0 -0
- data/lib/sidekiq/capistrano.rb +0 -32
- data/lib/sidekiq/extensions/action_mailer.rb +0 -26
- data/lib/sidekiq/extensions/active_record.rb +0 -27
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -21
- data/lib/sidekiq/middleware/client/unique_jobs.rb +0 -32
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/exception_handler.rb +0 -38
- data/lib/sidekiq/middleware/server/failure_jobs.rb +0 -24
- data/lib/sidekiq/middleware/server/logging.rb +0 -27
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -59
- data/lib/sidekiq/middleware/server/unique_jobs.rb +0 -15
- data/lib/sidekiq/retry.rb +0 -57
- data/lib/sidekiq/util.rb +0 -61
- data/lib/sidekiq/worker.rb +0 -37
- data/myapp/.gitignore +0 -15
- data/myapp/Capfile +0 -5
- data/myapp/Gemfile +0 -19
- data/myapp/Gemfile.lock +0 -143
- data/myapp/Rakefile +0 -7
- data/myapp/app/controllers/application_controller.rb +0 -3
- data/myapp/app/controllers/work_controller.rb +0 -38
- data/myapp/app/helpers/application_helper.rb +0 -2
- data/myapp/app/mailers/.gitkeep +0 -0
- data/myapp/app/mailers/user_mailer.rb +0 -9
- data/myapp/app/models/.gitkeep +0 -0
- data/myapp/app/models/post.rb +0 -5
- data/myapp/app/views/layouts/application.html.erb +0 -14
- data/myapp/app/views/user_mailer/greetings.html.erb +0 -3
- data/myapp/app/views/work/index.html.erb +0 -1
- data/myapp/app/workers/hard_worker.rb +0 -9
- data/myapp/config/application.rb +0 -59
- data/myapp/config/boot.rb +0 -6
- data/myapp/config/database.yml +0 -25
- data/myapp/config/deploy.rb +0 -15
- data/myapp/config/environment.rb +0 -5
- data/myapp/config/environments/development.rb +0 -38
- data/myapp/config/environments/production.rb +0 -67
- data/myapp/config/environments/test.rb +0 -37
- data/myapp/config/initializers/backtrace_silencers.rb +0 -7
- data/myapp/config/initializers/inflections.rb +0 -15
- data/myapp/config/initializers/mime_types.rb +0 -5
- data/myapp/config/initializers/secret_token.rb +0 -7
- data/myapp/config/initializers/session_store.rb +0 -8
- data/myapp/config/initializers/sidekiq.rb +0 -6
- data/myapp/config/initializers/wrap_parameters.rb +0 -14
- data/myapp/config/locales/en.yml +0 -5
- data/myapp/config/routes.rb +0 -10
- data/myapp/config.ru +0 -4
- data/myapp/db/migrate/20120123214055_create_posts.rb +0 -10
- data/myapp/db/seeds.rb +0 -7
- data/myapp/lib/assets/.gitkeep +0 -0
- data/myapp/lib/tasks/.gitkeep +0 -0
- data/myapp/log/.gitkeep +0 -0
- data/myapp/script/rails +0 -6
- data/test/config.yml +0 -9
- data/test/fake_env.rb +0 -0
- data/test/helper.rb +0 -15
- data/test/test_cli.rb +0 -168
- data/test/test_client.rb +0 -105
- data/test/test_extensions.rb +0 -68
- data/test/test_manager.rb +0 -43
- data/test/test_middleware.rb +0 -92
- data/test/test_processor.rb +0 -32
- data/test/test_retry.rb +0 -83
- data/test/test_stats.rb +0 -78
- data/test/test_testing.rb +0 -65
- data/test/test_web.rb +0 -61
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +0 -91
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +0 -98
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +0 -154
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +0 -136
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +0 -92
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +0 -210
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +0 -95
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +0 -125
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +0 -130
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +0 -270
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +0 -51
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +0 -271
- data/web/assets/javascripts/vendor/bootstrap.js +0 -12
- data/web/assets/javascripts/vendor/jquery.js +0 -9266
- data/web/assets/stylesheets/vendor/bootstrap-responsive.css +0 -567
- data/web/assets/stylesheets/vendor/bootstrap.css +0 -3365
- data/web/views/index.slim +0 -62
- data/web/views/layout.slim +0 -24
- data/web/views/queue.slim +0 -11
@@ -0,0 +1,131 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
orig_context = current.dup
|
10
|
+
current.merge!(hash)
|
11
|
+
yield
|
12
|
+
ensure
|
13
|
+
Thread.current[:sidekiq_context] = orig_context
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.current
|
17
|
+
Thread.current[:sidekiq_context] ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.add(k, v)
|
21
|
+
current[k] = v
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
module LoggingUtils
|
26
|
+
LEVELS = {
|
27
|
+
"debug" => 0,
|
28
|
+
"info" => 1,
|
29
|
+
"warn" => 2,
|
30
|
+
"error" => 3,
|
31
|
+
"fatal" => 4
|
32
|
+
}
|
33
|
+
LEVELS.default_proc = proc do |_, level|
|
34
|
+
puts("Invalid log level: #{level.inspect}")
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def local_level
|
45
|
+
Thread.current[:sidekiq_log_level]
|
46
|
+
end
|
47
|
+
|
48
|
+
def local_level=(level)
|
49
|
+
case level
|
50
|
+
when Integer
|
51
|
+
Thread.current[:sidekiq_log_level] = level
|
52
|
+
when Symbol, String
|
53
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
54
|
+
when nil
|
55
|
+
Thread.current[:sidekiq_log_level] = nil
|
56
|
+
else
|
57
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def level
|
62
|
+
local_level || super
|
63
|
+
end
|
64
|
+
|
65
|
+
# Change the thread-local level for the duration of the given block.
|
66
|
+
def log_at(level)
|
67
|
+
old_local_level = local_level
|
68
|
+
self.local_level = level
|
69
|
+
yield
|
70
|
+
ensure
|
71
|
+
self.local_level = old_local_level
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class Logger < ::Logger
|
76
|
+
include LoggingUtils
|
77
|
+
|
78
|
+
module Formatters
|
79
|
+
class Base < ::Logger::Formatter
|
80
|
+
def tid
|
81
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
82
|
+
end
|
83
|
+
|
84
|
+
def ctx
|
85
|
+
Sidekiq::Context.current
|
86
|
+
end
|
87
|
+
|
88
|
+
def format_context
|
89
|
+
if ctx.any?
|
90
|
+
" " + ctx.compact.map { |k, v|
|
91
|
+
case v
|
92
|
+
when Array
|
93
|
+
"#{k}=#{v.join(",")}"
|
94
|
+
else
|
95
|
+
"#{k}=#{v}"
|
96
|
+
end
|
97
|
+
}.join(" ")
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class Pretty < Base
|
103
|
+
def call(severity, time, program_name, message)
|
104
|
+
"#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
class WithoutTimestamp < Pretty
|
109
|
+
def call(severity, time, program_name, message)
|
110
|
+
"pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class JSON < Base
|
115
|
+
def call(severity, time, program_name, message)
|
116
|
+
hash = {
|
117
|
+
ts: time.utc.iso8601(3),
|
118
|
+
pid: ::Process.pid,
|
119
|
+
tid: tid,
|
120
|
+
lvl: severity,
|
121
|
+
msg: message
|
122
|
+
}
|
123
|
+
c = ctx
|
124
|
+
hash["ctx"] = c unless c.empty?
|
125
|
+
|
126
|
+
Sidekiq.dump_json(hash) << "\n"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,141 +1,134 @@
|
|
1
|
-
|
2
|
-
require 'redis'
|
3
|
-
require 'multi_json'
|
1
|
+
# frozen_string_literal: true
|
4
2
|
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require 'connection_pool/version'
|
3
|
+
require "sidekiq/processor"
|
4
|
+
require "set"
|
8
5
|
|
9
6
|
module Sidekiq
|
10
|
-
|
11
7
|
##
|
12
|
-
# The
|
13
|
-
#
|
14
|
-
#
|
8
|
+
# The Manager is the central coordination point in Sidekiq, controlling
|
9
|
+
# the lifecycle of the Processors.
|
10
|
+
#
|
11
|
+
# Tasks:
|
12
|
+
#
|
13
|
+
# 1. start: Spin up Processors.
|
14
|
+
# 3. processor_died: Handle job failure, throw away Processor, create new one.
|
15
|
+
# 4. quiet: shutdown idle Processors.
|
16
|
+
# 5. stop: hard stop the Processors by deadline.
|
17
|
+
#
|
18
|
+
# Note that only the last task requires its own Thread since it has to monitor
|
19
|
+
# the shutdown process. The other tasks are performed by other threads.
|
15
20
|
#
|
16
21
|
class Manager
|
17
|
-
include
|
18
|
-
include Celluloid
|
22
|
+
include Sidekiq::Component
|
19
23
|
|
20
|
-
|
24
|
+
attr_reader :workers
|
25
|
+
attr_reader :capsule
|
21
26
|
|
22
|
-
def initialize(
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
@count = options[:concurrency] || 25
|
27
|
-
@queues = options[:queues]
|
28
|
-
@done_callback = nil
|
27
|
+
def initialize(capsule)
|
28
|
+
@config = @capsule = capsule
|
29
|
+
@count = capsule.concurrency
|
30
|
+
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
29
31
|
|
30
32
|
@done = false
|
31
|
-
@
|
32
|
-
@
|
33
|
+
@workers = Set.new
|
34
|
+
@plock = Mutex.new
|
35
|
+
@count.times do
|
36
|
+
@workers << Processor.new(@config, &method(:processor_result))
|
37
|
+
end
|
33
38
|
end
|
34
39
|
|
35
|
-
def
|
36
|
-
|
37
|
-
|
40
|
+
def start
|
41
|
+
@workers.each(&:start)
|
42
|
+
end
|
38
43
|
|
44
|
+
def quiet
|
45
|
+
return if @done
|
39
46
|
@done = true
|
40
|
-
@ready.each { |x| x.terminate if x.alive? }
|
41
|
-
@ready.clear
|
42
47
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
conn.srem('workers', name) if name =~ /:#{process_id}-/
|
47
|
-
end
|
48
|
-
end
|
48
|
+
logger.info { "Terminating quiet threads for #{capsule.name} capsule" }
|
49
|
+
@workers.each(&:terminate)
|
50
|
+
end
|
49
51
|
|
50
|
-
|
51
|
-
|
52
|
-
# after(0) needed to avoid deadlock in Celluoid after USR1 + TERM
|
53
|
-
return after(0) { signal(:shutdown) }
|
54
|
-
else
|
55
|
-
logger.info { "Pausing #{timeout} seconds to allow workers to finish..." }
|
56
|
-
end
|
52
|
+
def stop(deadline)
|
53
|
+
quiet
|
57
54
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
end
|
55
|
+
# some of the shutdown events can be async,
|
56
|
+
# we don't have any way to know when they're done but
|
57
|
+
# give them a little time to take effect
|
58
|
+
sleep PAUSE_TIME
|
59
|
+
return if @workers.empty?
|
64
60
|
|
65
|
-
|
66
|
-
|
67
|
-
|
61
|
+
logger.info { "Pausing to allow jobs to finish..." }
|
62
|
+
wait_for(deadline) { @workers.empty? }
|
63
|
+
return if @workers.empty?
|
68
64
|
|
69
|
-
|
70
|
-
|
65
|
+
hard_shutdown
|
66
|
+
ensure
|
67
|
+
capsule.stop
|
71
68
|
end
|
72
69
|
|
73
|
-
def
|
74
|
-
|
75
|
-
@
|
76
|
-
@
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
else
|
81
|
-
@ready << processor if processor.alive?
|
70
|
+
def processor_result(processor, reason = nil)
|
71
|
+
@plock.synchronize do
|
72
|
+
@workers.delete(processor)
|
73
|
+
unless @done
|
74
|
+
p = Processor.new(@config, &method(:processor_result))
|
75
|
+
@workers << p
|
76
|
+
p.start
|
82
77
|
end
|
83
|
-
dispatch
|
84
78
|
end
|
85
79
|
end
|
86
80
|
|
87
|
-
def
|
88
|
-
@
|
89
|
-
|
90
|
-
unless stopped?
|
91
|
-
@ready << Processor.new_link(current_actor)
|
92
|
-
dispatch
|
93
|
-
else
|
94
|
-
signal(:shutdown) if @busy.empty?
|
95
|
-
end
|
81
|
+
def stopped?
|
82
|
+
@done
|
96
83
|
end
|
97
84
|
|
98
85
|
private
|
99
86
|
|
100
|
-
def
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
87
|
+
def hard_shutdown
|
88
|
+
# We've reached the timeout and we still have busy threads.
|
89
|
+
# They must die but their jobs shall live on.
|
90
|
+
cleanup = nil
|
91
|
+
@plock.synchronize do
|
92
|
+
cleanup = @workers.dup
|
106
93
|
end
|
107
|
-
!!msg
|
108
|
-
end
|
109
94
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
# processors somehow.
|
116
|
-
raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
|
117
|
-
|
118
|
-
# Dispatch loop
|
119
|
-
loop do
|
120
|
-
break logger.debug('no processors') if @ready.empty?
|
121
|
-
found = false
|
122
|
-
@ready.size.times do
|
123
|
-
found ||= find_work(@queues.sample)
|
124
|
-
end
|
125
|
-
break unless found
|
126
|
-
end
|
95
|
+
if cleanup.size > 0
|
96
|
+
jobs = cleanup.map { |p| p.job }.compact
|
97
|
+
|
98
|
+
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
99
|
+
logger.debug { "Jobs still in progress #{jobs.inspect}" }
|
127
100
|
|
128
|
-
#
|
129
|
-
#
|
130
|
-
#
|
131
|
-
|
132
|
-
|
133
|
-
|
101
|
+
# Re-enqueue unfinished jobs
|
102
|
+
# NOTE: You may notice that we may push a job back to redis before
|
103
|
+
# the thread is terminated. This is ok because Sidekiq's
|
104
|
+
# contract says that jobs are run AT LEAST once. Process termination
|
105
|
+
# is delayed until we're certain the jobs are back in Redis because
|
106
|
+
# it is worse to lose a job than to run it twice.
|
107
|
+
capsule.fetcher.bulk_requeue(jobs)
|
134
108
|
end
|
109
|
+
|
110
|
+
cleanup.each do |processor|
|
111
|
+
processor.kill
|
112
|
+
end
|
113
|
+
|
114
|
+
# when this method returns, we immediately call `exit` which may not give
|
115
|
+
# the remaining threads time to run `ensure` blocks, etc. We pause here up
|
116
|
+
# to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
|
117
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
118
|
+
wait_for(deadline) { @workers.empty? }
|
135
119
|
end
|
136
120
|
|
137
|
-
|
138
|
-
|
121
|
+
# hack for quicker development / testing environment #2774
|
122
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
123
|
+
|
124
|
+
# Wait for the orblock to be true or the deadline passed.
|
125
|
+
def wait_for(deadline, &condblock)
|
126
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
127
|
+
while remaining > PAUSE_TIME
|
128
|
+
return if condblock.call
|
129
|
+
sleep PAUSE_TIME
|
130
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
131
|
+
end
|
139
132
|
end
|
140
133
|
end
|
141
134
|
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
require "sidekiq"
|
2
|
+
require "date"
|
3
|
+
require "set"
|
4
|
+
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Metrics
|
9
|
+
# Allows caller to query for Sidekiq execution metrics within Redis.
|
10
|
+
# Caller sets a set of attributes to act as filters. {#fetch} will call
|
11
|
+
# Redis and return a Hash of results.
|
12
|
+
#
|
13
|
+
# NB: all metrics and times/dates are UTC only. We specifically do not
|
14
|
+
# support timezones.
|
15
|
+
class Query
|
16
|
+
def initialize(pool: nil, now: Time.now)
|
17
|
+
@time = now.utc
|
18
|
+
@pool = pool || Sidekiq.default_configuration.redis_pool
|
19
|
+
@klass = nil
|
20
|
+
end
|
21
|
+
|
22
|
+
# Get metric data for all jobs from the last hour
|
23
|
+
# +class_filter+: return only results for classes matching filter
|
24
|
+
def top_jobs(class_filter: nil, minutes: 60)
|
25
|
+
result = Result.new
|
26
|
+
|
27
|
+
time = @time
|
28
|
+
redis_results = @pool.with do |conn|
|
29
|
+
conn.pipelined do |pipe|
|
30
|
+
minutes.times do |idx|
|
31
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
32
|
+
pipe.hgetall key
|
33
|
+
result.prepend_bucket time
|
34
|
+
time -= 60
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
time = @time
|
40
|
+
redis_results.each do |hash|
|
41
|
+
hash.each do |k, v|
|
42
|
+
kls, metric = k.split("|")
|
43
|
+
next if class_filter && !class_filter.match?(kls)
|
44
|
+
result.job_results[kls].add_metric metric, time, v.to_i
|
45
|
+
end
|
46
|
+
time -= 60
|
47
|
+
end
|
48
|
+
|
49
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
50
|
+
|
51
|
+
result
|
52
|
+
end
|
53
|
+
|
54
|
+
def for_job(klass, minutes: 60)
|
55
|
+
result = Result.new
|
56
|
+
|
57
|
+
time = @time
|
58
|
+
redis_results = @pool.with do |conn|
|
59
|
+
conn.pipelined do |pipe|
|
60
|
+
minutes.times do |idx|
|
61
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
62
|
+
pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
|
63
|
+
result.prepend_bucket time
|
64
|
+
time -= 60
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
time = @time
|
70
|
+
@pool.with do |conn|
|
71
|
+
redis_results.each do |(ms, p, f)|
|
72
|
+
result.job_results[klass].add_metric "ms", time, ms.to_i if ms
|
73
|
+
result.job_results[klass].add_metric "p", time, p.to_i if p
|
74
|
+
result.job_results[klass].add_metric "f", time, f.to_i if f
|
75
|
+
result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
|
76
|
+
time -= 60
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
81
|
+
|
82
|
+
result
|
83
|
+
end
|
84
|
+
|
85
|
+
class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
|
86
|
+
def initialize
|
87
|
+
super
|
88
|
+
self.buckets = []
|
89
|
+
self.marks = []
|
90
|
+
self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
|
91
|
+
end
|
92
|
+
|
93
|
+
def prepend_bucket(time)
|
94
|
+
buckets.unshift time.strftime("%H:%M")
|
95
|
+
self.ends_at ||= time
|
96
|
+
self.starts_at = time
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
class JobResult < Struct.new(:series, :hist, :totals)
|
101
|
+
def initialize
|
102
|
+
super
|
103
|
+
self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
|
104
|
+
self.hist = Hash.new { |h, k| h[k] = [] }
|
105
|
+
self.totals = Hash.new(0)
|
106
|
+
end
|
107
|
+
|
108
|
+
def add_metric(metric, time, value)
|
109
|
+
totals[metric] += value
|
110
|
+
series[metric][time.strftime("%H:%M")] += value
|
111
|
+
|
112
|
+
# Include timing measurements in seconds for convenience
|
113
|
+
add_metric("s", time, value / 1000.0) if metric == "ms"
|
114
|
+
end
|
115
|
+
|
116
|
+
def add_hist(time, hist_result)
|
117
|
+
hist[time.strftime("%H:%M")] = hist_result
|
118
|
+
end
|
119
|
+
|
120
|
+
def total_avg(metric = "ms")
|
121
|
+
completed = totals["p"] - totals["f"]
|
122
|
+
totals[metric].to_f / completed
|
123
|
+
end
|
124
|
+
|
125
|
+
def series_avg(metric = "ms")
|
126
|
+
series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
|
127
|
+
completed = series.dig("p", bucket) - series.dig("f", bucket)
|
128
|
+
result[bucket] = (completed == 0) ? 0 : value.to_f / completed
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
class MarkResult < Struct.new(:time, :label)
|
134
|
+
def bucket
|
135
|
+
time.strftime("%H:%M")
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
private
|
140
|
+
|
141
|
+
def fetch_marks(time_range)
|
142
|
+
[].tap do |result|
|
143
|
+
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
|
144
|
+
|
145
|
+
marks.each do |timestamp, label|
|
146
|
+
time = Time.parse(timestamp)
|
147
|
+
if time_range.cover? time
|
148
|
+
result << MarkResult.new(time, label)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Metrics
|
5
|
+
# This is the only dependency on concurrent-ruby in Sidekiq but it's
|
6
|
+
# mandatory for thread-safety until MRI supports atomic operations on values.
|
7
|
+
Counter = ::Concurrent::AtomicFixnum
|
8
|
+
|
9
|
+
# Implements space-efficient but statistically useful histogram storage.
|
10
|
+
# A precise time histogram stores every time. Instead we break times into a set of
|
11
|
+
# known buckets and increment counts of the associated time bucket. Even if we call
|
12
|
+
# the histogram a million times, we'll still only store 26 buckets.
|
13
|
+
# NB: needs to be thread-safe or resiliant to races.
|
14
|
+
#
|
15
|
+
# To store this data, we use Redis' BITFIELD command to store unsigned 16-bit counters
|
16
|
+
# per bucket per klass per minute. It's unlikely that most people will be executing more
|
17
|
+
# than 1000 job/sec for a full minute of a specific type.
|
18
|
+
class Histogram
|
19
|
+
include Enumerable
|
20
|
+
|
21
|
+
# This number represents the maximum milliseconds for this bucket.
|
22
|
+
# 20 means all job executions up to 20ms, e.g. if a job takes
|
23
|
+
# 280ms, it'll increment bucket[7]. Note we can track job executions
|
24
|
+
# up to about 5.5 minutes. After that, it's assumed you're probably
|
25
|
+
# not too concerned with its performance.
|
26
|
+
BUCKET_INTERVALS = [
|
27
|
+
20, 30, 45, 65, 100,
|
28
|
+
150, 225, 335, 500, 750,
|
29
|
+
1100, 1700, 2500, 3800, 5750,
|
30
|
+
8500, 13000, 20000, 30000, 45000,
|
31
|
+
65000, 100000, 150000, 225000, 335000,
|
32
|
+
1e20 # the "maybe your job is too long" bucket
|
33
|
+
].freeze
|
34
|
+
LABELS = [
|
35
|
+
"20ms", "30ms", "45ms", "65ms", "100ms",
|
36
|
+
"150ms", "225ms", "335ms", "500ms", "750ms",
|
37
|
+
"1.1s", "1.7s", "2.5s", "3.8s", "5.75s",
|
38
|
+
"8.5s", "13s", "20s", "30s", "45s",
|
39
|
+
"65s", "100s", "150s", "225s", "335s",
|
40
|
+
"Slow"
|
41
|
+
].freeze
|
42
|
+
FETCH = "GET u16 #0 GET u16 #1 GET u16 #2 GET u16 #3 \
|
43
|
+
GET u16 #4 GET u16 #5 GET u16 #6 GET u16 #7 \
|
44
|
+
GET u16 #8 GET u16 #9 GET u16 #10 GET u16 #11 \
|
45
|
+
GET u16 #12 GET u16 #13 GET u16 #14 GET u16 #15 \
|
46
|
+
GET u16 #16 GET u16 #17 GET u16 #18 GET u16 #19 \
|
47
|
+
GET u16 #20 GET u16 #21 GET u16 #22 GET u16 #23 \
|
48
|
+
GET u16 #24 GET u16 #25".split
|
49
|
+
HISTOGRAM_TTL = 8 * 60 * 60
|
50
|
+
|
51
|
+
def each
|
52
|
+
buckets.each { |counter| yield counter.value }
|
53
|
+
end
|
54
|
+
|
55
|
+
def label(idx)
|
56
|
+
LABELS[idx]
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_reader :buckets
|
60
|
+
def initialize(klass)
|
61
|
+
@klass = klass
|
62
|
+
@buckets = Array.new(BUCKET_INTERVALS.size) { Counter.new }
|
63
|
+
end
|
64
|
+
|
65
|
+
def record_time(ms)
|
66
|
+
index_to_use = BUCKET_INTERVALS.each_index do |idx|
|
67
|
+
break idx if ms < BUCKET_INTERVALS[idx]
|
68
|
+
end
|
69
|
+
|
70
|
+
@buckets[index_to_use].increment
|
71
|
+
end
|
72
|
+
|
73
|
+
def fetch(conn, now = Time.now)
|
74
|
+
window = now.utc.strftime("%d-%H:%-M")
|
75
|
+
key = "#{@klass}-#{window}"
|
76
|
+
conn.bitfield_ro(key, *FETCH)
|
77
|
+
end
|
78
|
+
|
79
|
+
def persist(conn, now = Time.now)
|
80
|
+
buckets, @buckets = @buckets, []
|
81
|
+
window = now.utc.strftime("%d-%H:%-M")
|
82
|
+
key = "#{@klass}-#{window}"
|
83
|
+
cmd = [key, "OVERFLOW", "SAT"]
|
84
|
+
buckets.each_with_index do |counter, idx|
|
85
|
+
val = counter.value
|
86
|
+
cmd << "INCRBY" << "u16" << "##{idx}" << val.to_s if val > 0
|
87
|
+
end
|
88
|
+
|
89
|
+
conn.bitfield(*cmd) if cmd.size > 3
|
90
|
+
conn.expire(key, HISTOGRAM_TTL)
|
91
|
+
key
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|