sidekiq 0.10.0 → 7.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Changes.md +2047 -0
- data/LICENSE.txt +9 -0
- data/README.md +73 -27
- data/bin/sidekiq +25 -9
- data/bin/sidekiqload +247 -0
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/job.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +1145 -0
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +348 -109
- data/lib/sidekiq/client.rb +241 -41
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +88 -0
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +300 -0
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +271 -0
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +96 -103
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +149 -38
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +55 -0
- data/lib/sidekiq/processor.rb +246 -61
- data/lib/sidekiq/rails.rb +60 -13
- data/lib/sidekiq/redis_client_adapter.rb +95 -0
- data/lib/sidekiq/redis_connection.rb +68 -15
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +236 -0
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +30 -0
- data/lib/sidekiq/testing.rb +304 -10
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +4 -1
- data/lib/sidekiq/web/action.rb +93 -0
- data/lib/sidekiq/web/application.rb +447 -0
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +370 -0
- data/lib/sidekiq/web/router.rb +104 -0
- data/lib/sidekiq/web.rb +143 -74
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +120 -73
- data/sidekiq.gemspec +26 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status.png +0 -0
- data/web/assets/javascripts/application.js +162 -3
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +168 -0
- data/web/assets/javascripts/dashboard.js +59 -0
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +153 -0
- data/web/assets/stylesheets/application.css +720 -7
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +5 -0
- data/web/locales/ar.yml +87 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +75 -0
- data/web/locales/de.yml +81 -0
- data/web/locales/el.yml +87 -0
- data/web/locales/en.yml +101 -0
- data/web/locales/es.yml +86 -0
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +99 -0
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +80 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +91 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +96 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +83 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +77 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +95 -0
- data/web/locales/zh-tw.yml +102 -0
- data/web/views/_footer.erb +23 -0
- data/web/views/_job_info.erb +105 -0
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +52 -0
- data/web/views/_paging.erb +25 -0
- data/web/views/_poll_link.erb +4 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +148 -0
- data/web/views/dashboard.erb +115 -0
- data/web/views/dead.erb +34 -0
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +42 -0
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +74 -0
- data/web/views/queue.erb +55 -0
- data/web/views/queues.erb +40 -0
- data/web/views/retries.erb +79 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +56 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +159 -237
- data/.gitignore +0 -6
- data/.rvmrc +0 -4
- data/COMM-LICENSE +0 -75
- data/Gemfile +0 -10
- data/LICENSE +0 -22
- data/Rakefile +0 -9
- data/TODO.md +0 -1
- data/bin/client +0 -7
- data/bin/sidekiqctl +0 -43
- data/config.ru +0 -8
- data/examples/chef/cookbooks/sidekiq/README.rdoc +0 -11
- data/examples/chef/cookbooks/sidekiq/recipes/default.rb +0 -55
- data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb +0 -8
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb +0 -219
- data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb +0 -22
- data/examples/config.yml +0 -9
- data/examples/monitrc.conf +0 -6
- data/examples/por.rb +0 -27
- data/examples/scheduling.rb +0 -37
- data/examples/sinkiq.rb +0 -57
- data/examples/web-ui.png +0 -0
- data/lib/sidekiq/capistrano.rb +0 -32
- data/lib/sidekiq/extensions/action_mailer.rb +0 -26
- data/lib/sidekiq/extensions/active_record.rb +0 -27
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -21
- data/lib/sidekiq/middleware/client/unique_jobs.rb +0 -32
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/exception_handler.rb +0 -38
- data/lib/sidekiq/middleware/server/failure_jobs.rb +0 -24
- data/lib/sidekiq/middleware/server/logging.rb +0 -27
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -59
- data/lib/sidekiq/middleware/server/unique_jobs.rb +0 -15
- data/lib/sidekiq/retry.rb +0 -57
- data/lib/sidekiq/util.rb +0 -61
- data/lib/sidekiq/worker.rb +0 -37
- data/myapp/.gitignore +0 -15
- data/myapp/Capfile +0 -5
- data/myapp/Gemfile +0 -19
- data/myapp/Gemfile.lock +0 -143
- data/myapp/Rakefile +0 -7
- data/myapp/app/controllers/application_controller.rb +0 -3
- data/myapp/app/controllers/work_controller.rb +0 -38
- data/myapp/app/helpers/application_helper.rb +0 -2
- data/myapp/app/mailers/.gitkeep +0 -0
- data/myapp/app/mailers/user_mailer.rb +0 -9
- data/myapp/app/models/.gitkeep +0 -0
- data/myapp/app/models/post.rb +0 -5
- data/myapp/app/views/layouts/application.html.erb +0 -14
- data/myapp/app/views/user_mailer/greetings.html.erb +0 -3
- data/myapp/app/views/work/index.html.erb +0 -1
- data/myapp/app/workers/hard_worker.rb +0 -9
- data/myapp/config/application.rb +0 -59
- data/myapp/config/boot.rb +0 -6
- data/myapp/config/database.yml +0 -25
- data/myapp/config/deploy.rb +0 -15
- data/myapp/config/environment.rb +0 -5
- data/myapp/config/environments/development.rb +0 -38
- data/myapp/config/environments/production.rb +0 -67
- data/myapp/config/environments/test.rb +0 -37
- data/myapp/config/initializers/backtrace_silencers.rb +0 -7
- data/myapp/config/initializers/inflections.rb +0 -15
- data/myapp/config/initializers/mime_types.rb +0 -5
- data/myapp/config/initializers/secret_token.rb +0 -7
- data/myapp/config/initializers/session_store.rb +0 -8
- data/myapp/config/initializers/sidekiq.rb +0 -6
- data/myapp/config/initializers/wrap_parameters.rb +0 -14
- data/myapp/config/locales/en.yml +0 -5
- data/myapp/config/routes.rb +0 -10
- data/myapp/config.ru +0 -4
- data/myapp/db/migrate/20120123214055_create_posts.rb +0 -10
- data/myapp/db/seeds.rb +0 -7
- data/myapp/lib/assets/.gitkeep +0 -0
- data/myapp/lib/tasks/.gitkeep +0 -0
- data/myapp/log/.gitkeep +0 -0
- data/myapp/script/rails +0 -6
- data/test/config.yml +0 -9
- data/test/fake_env.rb +0 -0
- data/test/helper.rb +0 -15
- data/test/test_cli.rb +0 -168
- data/test/test_client.rb +0 -105
- data/test/test_extensions.rb +0 -68
- data/test/test_manager.rb +0 -43
- data/test/test_middleware.rb +0 -92
- data/test/test_processor.rb +0 -32
- data/test/test_retry.rb +0 -83
- data/test/test_stats.rb +0 -78
- data/test/test_testing.rb +0 -65
- data/test/test_web.rb +0 -61
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +0 -91
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +0 -98
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +0 -154
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +0 -136
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +0 -92
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +0 -210
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +0 -95
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +0 -125
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +0 -130
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +0 -270
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +0 -51
- data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +0 -271
- data/web/assets/javascripts/vendor/bootstrap.js +0 -12
- data/web/assets/javascripts/vendor/jquery.js +0 -9266
- data/web/assets/stylesheets/vendor/bootstrap-responsive.css +0 -567
- data/web/assets/stylesheets/vendor/bootstrap.css +0 -3365
- data/web/views/index.slim +0 -62
- data/web/views/layout.slim +0 -24
- data/web/views/queue.slim +0 -11
@@ -0,0 +1,131 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
orig_context = current.dup
|
10
|
+
current.merge!(hash)
|
11
|
+
yield
|
12
|
+
ensure
|
13
|
+
Thread.current[:sidekiq_context] = orig_context
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.current
|
17
|
+
Thread.current[:sidekiq_context] ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.add(k, v)
|
21
|
+
current[k] = v
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
module LoggingUtils
|
26
|
+
LEVELS = {
|
27
|
+
"debug" => 0,
|
28
|
+
"info" => 1,
|
29
|
+
"warn" => 2,
|
30
|
+
"error" => 3,
|
31
|
+
"fatal" => 4
|
32
|
+
}
|
33
|
+
LEVELS.default_proc = proc do |_, level|
|
34
|
+
puts("Invalid log level: #{level.inspect}")
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def local_level
|
45
|
+
Thread.current[:sidekiq_log_level]
|
46
|
+
end
|
47
|
+
|
48
|
+
def local_level=(level)
|
49
|
+
case level
|
50
|
+
when Integer
|
51
|
+
Thread.current[:sidekiq_log_level] = level
|
52
|
+
when Symbol, String
|
53
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
54
|
+
when nil
|
55
|
+
Thread.current[:sidekiq_log_level] = nil
|
56
|
+
else
|
57
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def level
|
62
|
+
local_level || super
|
63
|
+
end
|
64
|
+
|
65
|
+
# Change the thread-local level for the duration of the given block.
|
66
|
+
def log_at(level)
|
67
|
+
old_local_level = local_level
|
68
|
+
self.local_level = level
|
69
|
+
yield
|
70
|
+
ensure
|
71
|
+
self.local_level = old_local_level
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class Logger < ::Logger
|
76
|
+
include LoggingUtils
|
77
|
+
|
78
|
+
module Formatters
|
79
|
+
class Base < ::Logger::Formatter
|
80
|
+
def tid
|
81
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
82
|
+
end
|
83
|
+
|
84
|
+
def ctx
|
85
|
+
Sidekiq::Context.current
|
86
|
+
end
|
87
|
+
|
88
|
+
def format_context
|
89
|
+
if ctx.any?
|
90
|
+
" " + ctx.compact.map { |k, v|
|
91
|
+
case v
|
92
|
+
when Array
|
93
|
+
"#{k}=#{v.join(",")}"
|
94
|
+
else
|
95
|
+
"#{k}=#{v}"
|
96
|
+
end
|
97
|
+
}.join(" ")
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class Pretty < Base
|
103
|
+
def call(severity, time, program_name, message)
|
104
|
+
"#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
class WithoutTimestamp < Pretty
|
109
|
+
def call(severity, time, program_name, message)
|
110
|
+
"pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class JSON < Base
|
115
|
+
def call(severity, time, program_name, message)
|
116
|
+
hash = {
|
117
|
+
ts: time.utc.iso8601(3),
|
118
|
+
pid: ::Process.pid,
|
119
|
+
tid: tid,
|
120
|
+
lvl: severity,
|
121
|
+
msg: message
|
122
|
+
}
|
123
|
+
c = ctx
|
124
|
+
hash["ctx"] = c unless c.empty?
|
125
|
+
|
126
|
+
Sidekiq.dump_json(hash) << "\n"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,141 +1,134 @@
|
|
1
|
-
|
2
|
-
require 'redis'
|
3
|
-
require 'multi_json'
|
1
|
+
# frozen_string_literal: true
|
4
2
|
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require 'connection_pool/version'
|
3
|
+
require "sidekiq/processor"
|
4
|
+
require "set"
|
8
5
|
|
9
6
|
module Sidekiq
|
10
|
-
|
11
7
|
##
|
12
|
-
# The
|
13
|
-
#
|
14
|
-
#
|
8
|
+
# The Manager is the central coordination point in Sidekiq, controlling
|
9
|
+
# the lifecycle of the Processors.
|
10
|
+
#
|
11
|
+
# Tasks:
|
12
|
+
#
|
13
|
+
# 1. start: Spin up Processors.
|
14
|
+
# 3. processor_died: Handle job failure, throw away Processor, create new one.
|
15
|
+
# 4. quiet: shutdown idle Processors.
|
16
|
+
# 5. stop: hard stop the Processors by deadline.
|
17
|
+
#
|
18
|
+
# Note that only the last task requires its own Thread since it has to monitor
|
19
|
+
# the shutdown process. The other tasks are performed by other threads.
|
15
20
|
#
|
16
21
|
class Manager
|
17
|
-
include
|
18
|
-
include Celluloid
|
22
|
+
include Sidekiq::Component
|
19
23
|
|
20
|
-
|
24
|
+
attr_reader :workers
|
25
|
+
attr_reader :capsule
|
21
26
|
|
22
|
-
def initialize(
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
@count = options[:concurrency] || 25
|
27
|
-
@queues = options[:queues]
|
28
|
-
@done_callback = nil
|
27
|
+
def initialize(capsule)
|
28
|
+
@config = @capsule = capsule
|
29
|
+
@count = capsule.concurrency
|
30
|
+
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
29
31
|
|
30
32
|
@done = false
|
31
|
-
@
|
32
|
-
@
|
33
|
+
@workers = Set.new
|
34
|
+
@plock = Mutex.new
|
35
|
+
@count.times do
|
36
|
+
@workers << Processor.new(@config, &method(:processor_result))
|
37
|
+
end
|
33
38
|
end
|
34
39
|
|
35
|
-
def
|
36
|
-
|
37
|
-
|
40
|
+
def start
|
41
|
+
@workers.each(&:start)
|
42
|
+
end
|
38
43
|
|
44
|
+
def quiet
|
45
|
+
return if @done
|
39
46
|
@done = true
|
40
|
-
@ready.each { |x| x.terminate if x.alive? }
|
41
|
-
@ready.clear
|
42
47
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
conn.srem('workers', name) if name =~ /:#{process_id}-/
|
47
|
-
end
|
48
|
-
end
|
48
|
+
logger.info { "Terminating quiet threads for #{capsule.name} capsule" }
|
49
|
+
@workers.each(&:terminate)
|
50
|
+
end
|
49
51
|
|
50
|
-
|
51
|
-
|
52
|
-
# after(0) needed to avoid deadlock in Celluoid after USR1 + TERM
|
53
|
-
return after(0) { signal(:shutdown) }
|
54
|
-
else
|
55
|
-
logger.info { "Pausing #{timeout} seconds to allow workers to finish..." }
|
56
|
-
end
|
52
|
+
def stop(deadline)
|
53
|
+
quiet
|
57
54
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
end
|
55
|
+
# some of the shutdown events can be async,
|
56
|
+
# we don't have any way to know when they're done but
|
57
|
+
# give them a little time to take effect
|
58
|
+
sleep PAUSE_TIME
|
59
|
+
return if @workers.empty?
|
64
60
|
|
65
|
-
|
66
|
-
|
67
|
-
|
61
|
+
logger.info { "Pausing to allow jobs to finish..." }
|
62
|
+
wait_for(deadline) { @workers.empty? }
|
63
|
+
return if @workers.empty?
|
68
64
|
|
69
|
-
|
70
|
-
|
65
|
+
hard_shutdown
|
66
|
+
ensure
|
67
|
+
capsule.stop
|
71
68
|
end
|
72
69
|
|
73
|
-
def
|
74
|
-
|
75
|
-
@
|
76
|
-
@
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
else
|
81
|
-
@ready << processor if processor.alive?
|
70
|
+
def processor_result(processor, reason = nil)
|
71
|
+
@plock.synchronize do
|
72
|
+
@workers.delete(processor)
|
73
|
+
unless @done
|
74
|
+
p = Processor.new(@config, &method(:processor_result))
|
75
|
+
@workers << p
|
76
|
+
p.start
|
82
77
|
end
|
83
|
-
dispatch
|
84
78
|
end
|
85
79
|
end
|
86
80
|
|
87
|
-
def
|
88
|
-
@
|
89
|
-
|
90
|
-
unless stopped?
|
91
|
-
@ready << Processor.new_link(current_actor)
|
92
|
-
dispatch
|
93
|
-
else
|
94
|
-
signal(:shutdown) if @busy.empty?
|
95
|
-
end
|
81
|
+
def stopped?
|
82
|
+
@done
|
96
83
|
end
|
97
84
|
|
98
85
|
private
|
99
86
|
|
100
|
-
def
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
87
|
+
def hard_shutdown
|
88
|
+
# We've reached the timeout and we still have busy threads.
|
89
|
+
# They must die but their jobs shall live on.
|
90
|
+
cleanup = nil
|
91
|
+
@plock.synchronize do
|
92
|
+
cleanup = @workers.dup
|
106
93
|
end
|
107
|
-
!!msg
|
108
|
-
end
|
109
94
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
# processors somehow.
|
116
|
-
raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
|
117
|
-
|
118
|
-
# Dispatch loop
|
119
|
-
loop do
|
120
|
-
break logger.debug('no processors') if @ready.empty?
|
121
|
-
found = false
|
122
|
-
@ready.size.times do
|
123
|
-
found ||= find_work(@queues.sample)
|
124
|
-
end
|
125
|
-
break unless found
|
126
|
-
end
|
95
|
+
if cleanup.size > 0
|
96
|
+
jobs = cleanup.map { |p| p.job }.compact
|
97
|
+
|
98
|
+
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
99
|
+
logger.debug { "Jobs still in progress #{jobs.inspect}" }
|
127
100
|
|
128
|
-
#
|
129
|
-
#
|
130
|
-
#
|
131
|
-
|
132
|
-
|
133
|
-
|
101
|
+
# Re-enqueue unfinished jobs
|
102
|
+
# NOTE: You may notice that we may push a job back to redis before
|
103
|
+
# the thread is terminated. This is ok because Sidekiq's
|
104
|
+
# contract says that jobs are run AT LEAST once. Process termination
|
105
|
+
# is delayed until we're certain the jobs are back in Redis because
|
106
|
+
# it is worse to lose a job than to run it twice.
|
107
|
+
capsule.fetcher.bulk_requeue(jobs)
|
134
108
|
end
|
109
|
+
|
110
|
+
cleanup.each do |processor|
|
111
|
+
processor.kill
|
112
|
+
end
|
113
|
+
|
114
|
+
# when this method returns, we immediately call `exit` which may not give
|
115
|
+
# the remaining threads time to run `ensure` blocks, etc. We pause here up
|
116
|
+
# to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
|
117
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
118
|
+
wait_for(deadline) { @workers.empty? }
|
135
119
|
end
|
136
120
|
|
137
|
-
|
138
|
-
|
121
|
+
# hack for quicker development / testing environment #2774
|
122
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
123
|
+
|
124
|
+
# Wait for the orblock to be true or the deadline passed.
|
125
|
+
def wait_for(deadline, &condblock)
|
126
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
127
|
+
while remaining > PAUSE_TIME
|
128
|
+
return if condblock.call
|
129
|
+
sleep PAUSE_TIME
|
130
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
131
|
+
end
|
139
132
|
end
|
140
133
|
end
|
141
134
|
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
require "sidekiq"
|
2
|
+
require "date"
|
3
|
+
require "set"
|
4
|
+
|
5
|
+
require "sidekiq/metrics/shared"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Metrics
|
9
|
+
# Allows caller to query for Sidekiq execution metrics within Redis.
|
10
|
+
# Caller sets a set of attributes to act as filters. {#fetch} will call
|
11
|
+
# Redis and return a Hash of results.
|
12
|
+
#
|
13
|
+
# NB: all metrics and times/dates are UTC only. We specifically do not
|
14
|
+
# support timezones.
|
15
|
+
class Query
|
16
|
+
def initialize(pool: nil, now: Time.now)
|
17
|
+
@time = now.utc
|
18
|
+
@pool = pool || Sidekiq.default_configuration.redis_pool
|
19
|
+
@klass = nil
|
20
|
+
end
|
21
|
+
|
22
|
+
# Get metric data for all jobs from the last hour
|
23
|
+
def top_jobs(minutes: 60)
|
24
|
+
result = Result.new
|
25
|
+
|
26
|
+
time = @time
|
27
|
+
redis_results = @pool.with do |conn|
|
28
|
+
conn.pipelined do |pipe|
|
29
|
+
minutes.times do |idx|
|
30
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
31
|
+
pipe.hgetall key
|
32
|
+
result.prepend_bucket time
|
33
|
+
time -= 60
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
time = @time
|
39
|
+
redis_results.each do |hash|
|
40
|
+
hash.each do |k, v|
|
41
|
+
kls, metric = k.split("|")
|
42
|
+
result.job_results[kls].add_metric metric, time, v.to_i
|
43
|
+
end
|
44
|
+
time -= 60
|
45
|
+
end
|
46
|
+
|
47
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
48
|
+
|
49
|
+
result
|
50
|
+
end
|
51
|
+
|
52
|
+
def for_job(klass, minutes: 60)
|
53
|
+
result = Result.new
|
54
|
+
|
55
|
+
time = @time
|
56
|
+
redis_results = @pool.with do |conn|
|
57
|
+
conn.pipelined do |pipe|
|
58
|
+
minutes.times do |idx|
|
59
|
+
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
|
60
|
+
pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
|
61
|
+
result.prepend_bucket time
|
62
|
+
time -= 60
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
time = @time
|
68
|
+
@pool.with do |conn|
|
69
|
+
redis_results.each do |(ms, p, f)|
|
70
|
+
result.job_results[klass].add_metric "ms", time, ms.to_i if ms
|
71
|
+
result.job_results[klass].add_metric "p", time, p.to_i if p
|
72
|
+
result.job_results[klass].add_metric "f", time, f.to_i if f
|
73
|
+
result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
|
74
|
+
time -= 60
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
result.marks = fetch_marks(result.starts_at..result.ends_at)
|
79
|
+
|
80
|
+
result
|
81
|
+
end
|
82
|
+
|
83
|
+
class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
|
84
|
+
def initialize
|
85
|
+
super
|
86
|
+
self.buckets = []
|
87
|
+
self.marks = []
|
88
|
+
self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
|
89
|
+
end
|
90
|
+
|
91
|
+
def prepend_bucket(time)
|
92
|
+
buckets.unshift time.strftime("%H:%M")
|
93
|
+
self.ends_at ||= time
|
94
|
+
self.starts_at = time
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class JobResult < Struct.new(:series, :hist, :totals)
|
99
|
+
def initialize
|
100
|
+
super
|
101
|
+
self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
|
102
|
+
self.hist = Hash.new { |h, k| h[k] = [] }
|
103
|
+
self.totals = Hash.new(0)
|
104
|
+
end
|
105
|
+
|
106
|
+
def add_metric(metric, time, value)
|
107
|
+
totals[metric] += value
|
108
|
+
series[metric][time.strftime("%H:%M")] += value
|
109
|
+
|
110
|
+
# Include timing measurements in seconds for convenience
|
111
|
+
add_metric("s", time, value / 1000.0) if metric == "ms"
|
112
|
+
end
|
113
|
+
|
114
|
+
def add_hist(time, hist_result)
|
115
|
+
hist[time.strftime("%H:%M")] = hist_result
|
116
|
+
end
|
117
|
+
|
118
|
+
def total_avg(metric = "ms")
|
119
|
+
completed = totals["p"] - totals["f"]
|
120
|
+
totals[metric].to_f / completed
|
121
|
+
end
|
122
|
+
|
123
|
+
def series_avg(metric = "ms")
|
124
|
+
series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
|
125
|
+
completed = series.dig("p", bucket) - series.dig("f", bucket)
|
126
|
+
result[bucket] = (completed == 0) ? 0 : value.to_f / completed
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
class MarkResult < Struct.new(:time, :label)
|
132
|
+
def bucket
|
133
|
+
time.strftime("%H:%M")
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
private
|
138
|
+
|
139
|
+
def fetch_marks(time_range)
|
140
|
+
[].tap do |result|
|
141
|
+
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
|
142
|
+
|
143
|
+
marks.each do |timestamp, label|
|
144
|
+
time = Time.parse(timestamp)
|
145
|
+
if time_range.cover? time
|
146
|
+
result << MarkResult.new(time, label)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Metrics
|
5
|
+
# This is the only dependency on concurrent-ruby in Sidekiq but it's
|
6
|
+
# mandatory for thread-safety until MRI supports atomic operations on values.
|
7
|
+
Counter = ::Concurrent::AtomicFixnum
|
8
|
+
|
9
|
+
# Implements space-efficient but statistically useful histogram storage.
|
10
|
+
# A precise time histogram stores every time. Instead we break times into a set of
|
11
|
+
# known buckets and increment counts of the associated time bucket. Even if we call
|
12
|
+
# the histogram a million times, we'll still only store 26 buckets.
|
13
|
+
# NB: needs to be thread-safe or resiliant to races.
|
14
|
+
#
|
15
|
+
# To store this data, we use Redis' BITFIELD command to store unsigned 16-bit counters
|
16
|
+
# per bucket per klass per minute. It's unlikely that most people will be executing more
|
17
|
+
# than 1000 job/sec for a full minute of a specific type.
|
18
|
+
class Histogram
|
19
|
+
include Enumerable
|
20
|
+
|
21
|
+
# This number represents the maximum milliseconds for this bucket.
|
22
|
+
# 20 means all job executions up to 20ms, e.g. if a job takes
|
23
|
+
# 280ms, it'll increment bucket[7]. Note we can track job executions
|
24
|
+
# up to about 5.5 minutes. After that, it's assumed you're probably
|
25
|
+
# not too concerned with its performance.
|
26
|
+
BUCKET_INTERVALS = [
|
27
|
+
20, 30, 45, 65, 100,
|
28
|
+
150, 225, 335, 500, 750,
|
29
|
+
1100, 1700, 2500, 3800, 5750,
|
30
|
+
8500, 13000, 20000, 30000, 45000,
|
31
|
+
65000, 100000, 150000, 225000, 335000,
|
32
|
+
1e20 # the "maybe your job is too long" bucket
|
33
|
+
].freeze
|
34
|
+
LABELS = [
|
35
|
+
"20ms", "30ms", "45ms", "65ms", "100ms",
|
36
|
+
"150ms", "225ms", "335ms", "500ms", "750ms",
|
37
|
+
"1.1s", "1.7s", "2.5s", "3.8s", "5.75s",
|
38
|
+
"8.5s", "13s", "20s", "30s", "45s",
|
39
|
+
"65s", "100s", "150s", "225s", "335s",
|
40
|
+
"Slow"
|
41
|
+
].freeze
|
42
|
+
FETCH = "GET u16 #0 GET u16 #1 GET u16 #2 GET u16 #3 \
|
43
|
+
GET u16 #4 GET u16 #5 GET u16 #6 GET u16 #7 \
|
44
|
+
GET u16 #8 GET u16 #9 GET u16 #10 GET u16 #11 \
|
45
|
+
GET u16 #12 GET u16 #13 GET u16 #14 GET u16 #15 \
|
46
|
+
GET u16 #16 GET u16 #17 GET u16 #18 GET u16 #19 \
|
47
|
+
GET u16 #20 GET u16 #21 GET u16 #22 GET u16 #23 \
|
48
|
+
GET u16 #24 GET u16 #25".split
|
49
|
+
HISTOGRAM_TTL = 8 * 60 * 60
|
50
|
+
|
51
|
+
def each
|
52
|
+
buckets.each { |counter| yield counter.value }
|
53
|
+
end
|
54
|
+
|
55
|
+
def label(idx)
|
56
|
+
LABELS[idx]
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_reader :buckets
|
60
|
+
def initialize(klass)
|
61
|
+
@klass = klass
|
62
|
+
@buckets = Array.new(BUCKET_INTERVALS.size) { Counter.new }
|
63
|
+
end
|
64
|
+
|
65
|
+
def record_time(ms)
|
66
|
+
index_to_use = BUCKET_INTERVALS.each_index do |idx|
|
67
|
+
break idx if ms < BUCKET_INTERVALS[idx]
|
68
|
+
end
|
69
|
+
|
70
|
+
@buckets[index_to_use].increment
|
71
|
+
end
|
72
|
+
|
73
|
+
def fetch(conn, now = Time.now)
|
74
|
+
window = now.utc.strftime("%d-%H:%-M")
|
75
|
+
key = "#{@klass}-#{window}"
|
76
|
+
conn.bitfield_ro(key, *FETCH)
|
77
|
+
end
|
78
|
+
|
79
|
+
def persist(conn, now = Time.now)
|
80
|
+
buckets, @buckets = @buckets, []
|
81
|
+
window = now.utc.strftime("%d-%H:%-M")
|
82
|
+
key = "#{@klass}-#{window}"
|
83
|
+
cmd = [key, "OVERFLOW", "SAT"]
|
84
|
+
buckets.each_with_index do |counter, idx|
|
85
|
+
val = counter.value
|
86
|
+
cmd << "INCRBY" << "u16" << "##{idx}" << val.to_s if val > 0
|
87
|
+
end
|
88
|
+
|
89
|
+
conn.bitfield(*cmd) if cmd.size > 3
|
90
|
+
conn.expire(key, HISTOGRAM_TTL)
|
91
|
+
key
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|