rails-autoscale-core 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +36 -0
- data/Rakefile +12 -0
- data/lib/rails-autoscale-core.rb +40 -0
- data/lib/rails_autoscale/adapter_api.rb +58 -0
- data/lib/rails_autoscale/config.rb +114 -0
- data/lib/rails_autoscale/job_metrics_collector/active_record_helper.rb +39 -0
- data/lib/rails_autoscale/job_metrics_collector.rb +105 -0
- data/lib/rails_autoscale/logger.rb +38 -0
- data/lib/rails_autoscale/metric.rb +11 -0
- data/lib/rails_autoscale/metrics_collector.rb +13 -0
- data/lib/rails_autoscale/metrics_store.rb +41 -0
- data/lib/rails_autoscale/report.rb +30 -0
- data/lib/rails_autoscale/reporter.rb +101 -0
- data/lib/rails_autoscale/request_metrics.rb +42 -0
- data/lib/rails_autoscale/request_middleware.rb +44 -0
- data/lib/rails_autoscale/version.rb +5 -0
- data/lib/rails_autoscale/web_metrics_collector.rb +16 -0
- data/rails-autoscale-core.gemspec +27 -0
- metadata +69 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 957a594b6d2686be815ed71d6c84ffcc92e14a14f5a9035fb12f92d6f979a400
|
4
|
+
data.tar.gz: b8cb713fb2731acbc11724097425d497c1f48e5bacaeff5cc41410a8b031f0a3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4b79400a71ac5f68401be26efa1bcc9e723b37d55c23802fb0ec30a9d1963bc892ac7b0c8067f11cddc802ffbd3dffd2cc80d7895ab7a1be89d0a7aebcff5232
|
7
|
+
data.tar.gz: f8940544874532959ae2ab3d5405c7087023a742a4f2a365d6ae4a2b15a602e6a41f9c6862d3171b55e4000e7c08008d37781f77531c0c492ea69c926a0377c1
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
rails-autoscale-core (1.0.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
addressable (2.8.0)
|
10
|
+
public_suffix (>= 2.0.2, < 5.0)
|
11
|
+
crack (0.4.5)
|
12
|
+
rexml
|
13
|
+
hashdiff (1.0.1)
|
14
|
+
minitest (5.15.0)
|
15
|
+
public_suffix (4.0.6)
|
16
|
+
rake (13.0.6)
|
17
|
+
rexml (3.2.5)
|
18
|
+
webmock (3.14.0)
|
19
|
+
addressable (>= 2.8.0)
|
20
|
+
crack (>= 0.3.2)
|
21
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
22
|
+
|
23
|
+
PLATFORMS
|
24
|
+
arm64-darwin-20
|
25
|
+
arm64-darwin-21
|
26
|
+
x86_64-darwin-21
|
27
|
+
x86_64-linux
|
28
|
+
|
29
|
+
DEPENDENCIES
|
30
|
+
minitest
|
31
|
+
rails-autoscale-core!
|
32
|
+
rake (>= 12.3.3)
|
33
|
+
webmock
|
34
|
+
|
35
|
+
BUNDLED WITH
|
36
|
+
2.3.9
|
data/Rakefile
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/config"
|
4
|
+
require "rails_autoscale/version"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
# Allows configuring Rails Autoscale through a block, usually defined during application initialization.
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
#
|
11
|
+
# RailsAutoscale.configure do |config|
|
12
|
+
# config.logger = MyLogger.new
|
13
|
+
# end
|
14
|
+
def self.configure
|
15
|
+
yield Config.instance
|
16
|
+
end
|
17
|
+
|
18
|
+
@adapters = []
|
19
|
+
class << self
|
20
|
+
attr_reader :adapters
|
21
|
+
end
|
22
|
+
|
23
|
+
Adapter = Struct.new(:identifier, :adapter_info, :metrics_collector) do
|
24
|
+
def as_json
|
25
|
+
{identifier => adapter_info}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.add_adapter(identifier, adapter_info, metrics_collector: nil, expose_config: nil)
|
30
|
+
Config.expose_adapter_config(expose_config) if expose_config
|
31
|
+
@adapters << Adapter.new(identifier, adapter_info, metrics_collector)
|
32
|
+
end
|
33
|
+
|
34
|
+
add_adapter :"rails-autoscale-core", {
|
35
|
+
adapter_version: VERSION,
|
36
|
+
language_version: RUBY_VERSION
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
Judoscale = RailsAutoscale
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "net/http"
|
4
|
+
require "uri"
|
5
|
+
require "json"
|
6
|
+
require "rails_autoscale/logger"
|
7
|
+
|
8
|
+
module RailsAutoscale
|
9
|
+
class AdapterApi
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
SUCCESS = "success"
|
13
|
+
|
14
|
+
def initialize(config)
|
15
|
+
@config = config
|
16
|
+
end
|
17
|
+
|
18
|
+
def report_metrics(report_json)
|
19
|
+
post_json "/v3/reports", report_json
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def post_json(path, data)
|
25
|
+
headers = {"Content-Type" => "application/json"}
|
26
|
+
post_raw path: path, body: JSON.dump(data), headers: headers
|
27
|
+
end
|
28
|
+
|
29
|
+
def post_raw(options)
|
30
|
+
uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
|
31
|
+
ssl = uri.scheme == "https"
|
32
|
+
|
33
|
+
response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
|
34
|
+
request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
|
35
|
+
request.body = options.fetch(:body)
|
36
|
+
|
37
|
+
logger.debug "Posting #{request.body.size} bytes to #{uri}"
|
38
|
+
http.request(request)
|
39
|
+
end
|
40
|
+
|
41
|
+
case response.code.to_i
|
42
|
+
when 200...300 then SuccessResponse.new(response.body)
|
43
|
+
else FailureResponse.new([response.code, response.message].join(" - "))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class SuccessResponse < Struct.new(:body)
|
48
|
+
def data
|
49
|
+
JSON.parse(body)
|
50
|
+
rescue TypeError
|
51
|
+
{}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class FailureResponse < Struct.new(:failure_message)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
class Config
|
8
|
+
class Dyno
|
9
|
+
attr_reader :name, :num
|
10
|
+
|
11
|
+
def initialize(dyno_string)
|
12
|
+
@name, @num = dyno_string.to_s.split(".")
|
13
|
+
@num = @num.to_i
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_s
|
17
|
+
"#{name}.#{num}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class JobAdapterConfig
|
22
|
+
UUID_REGEXP = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/
|
23
|
+
DEFAULT_QUEUE_FILTER = ->(queue_name) { !UUID_REGEXP.match?(queue_name) }
|
24
|
+
|
25
|
+
attr_accessor :identifier, :enabled, :max_queues, :queues, :queue_filter, :track_busy_jobs
|
26
|
+
|
27
|
+
def initialize(identifier)
|
28
|
+
@identifier = identifier
|
29
|
+
reset
|
30
|
+
end
|
31
|
+
|
32
|
+
def reset
|
33
|
+
@enabled = true
|
34
|
+
@max_queues = 20
|
35
|
+
@queues = []
|
36
|
+
@queue_filter = DEFAULT_QUEUE_FILTER
|
37
|
+
@track_busy_jobs = false
|
38
|
+
end
|
39
|
+
|
40
|
+
def as_json
|
41
|
+
{
|
42
|
+
identifier => {
|
43
|
+
max_queues: max_queues,
|
44
|
+
queues: queues,
|
45
|
+
queue_filter: queue_filter != DEFAULT_QUEUE_FILTER,
|
46
|
+
track_busy_jobs: track_busy_jobs
|
47
|
+
}
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
include Singleton
|
53
|
+
|
54
|
+
@adapter_configs = []
|
55
|
+
class << self
|
56
|
+
attr_reader :adapter_configs
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.expose_adapter_config(config_instance)
|
60
|
+
adapter_configs << config_instance
|
61
|
+
|
62
|
+
define_method(config_instance.identifier) do
|
63
|
+
config_instance
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
attr_accessor :api_base_url, :report_interval_seconds, :max_request_size_bytes, :logger, :log_tag
|
68
|
+
attr_reader :dyno, :log_level
|
69
|
+
|
70
|
+
def initialize
|
71
|
+
reset
|
72
|
+
end
|
73
|
+
|
74
|
+
def reset
|
75
|
+
# Allow the API URL to be configured - needed for testing.
|
76
|
+
@api_base_url = ENV["RAILS_AUTOSCALE_URL"] || ENV["JUDOSCALE_URL"]
|
77
|
+
@log_tag = ENV["JUDOSCALE_URL"] ? "Judoscale" : "RailsAutoscale"
|
78
|
+
self.dyno = ENV["DYNO"]
|
79
|
+
@max_request_size_bytes = 100_000 # ignore request payloads over 100k since they skew the queue times
|
80
|
+
@report_interval_seconds = 10
|
81
|
+
self.log_level = ENV["RAILS_AUTOSCALE_LOG_LEVEL"] || ENV["JUDOSCALE_LOG_LEVEL"]
|
82
|
+
@logger = ::Logger.new($stdout)
|
83
|
+
|
84
|
+
self.class.adapter_configs.each(&:reset)
|
85
|
+
end
|
86
|
+
|
87
|
+
def dyno=(dyno_string)
|
88
|
+
@dyno = Dyno.new(dyno_string)
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_level=(new_level)
|
92
|
+
@log_level = new_level ? ::Logger::Severity.const_get(new_level.to_s.upcase) : nil
|
93
|
+
end
|
94
|
+
|
95
|
+
def as_json
|
96
|
+
adapter_configs_json = self.class.adapter_configs.reduce({}) { |hash, config| hash.merge!(config.as_json) }
|
97
|
+
|
98
|
+
{
|
99
|
+
log_level: log_level,
|
100
|
+
logger: logger.class.name,
|
101
|
+
report_interval_seconds: report_interval_seconds,
|
102
|
+
max_request_size_bytes: max_request_size_bytes
|
103
|
+
}.merge!(adapter_configs_json)
|
104
|
+
end
|
105
|
+
|
106
|
+
def to_s
|
107
|
+
"#{@dyno}##{Process.pid}"
|
108
|
+
end
|
109
|
+
|
110
|
+
def ignore_large_requests?
|
111
|
+
@max_request_size_bytes
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class JobMetricsCollector
|
5
|
+
module ActiveRecordHelper
|
6
|
+
# Cleanup any whitespace characters (including new lines) from the SQL for simpler logging.
|
7
|
+
# Reference: ActiveSupport's `squish!` method. https://api.rubyonrails.org/classes/String.html#method-i-squish
|
8
|
+
def self.cleanse_sql(sql)
|
9
|
+
sql = sql.dup
|
10
|
+
sql.gsub!(/[[:space:]]+/, " ")
|
11
|
+
sql.strip!
|
12
|
+
sql
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def select_rows_silently(sql)
|
18
|
+
if Config.instance.log_level && ::ActiveRecord::Base.logger.respond_to?(:silence)
|
19
|
+
::ActiveRecord::Base.logger.silence(Config.instance.log_level) { select_rows_tagged(sql) }
|
20
|
+
else
|
21
|
+
select_rows_tagged(sql)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def select_rows_tagged(sql)
|
26
|
+
if ActiveRecord::Base.logger.respond_to?(:tagged)
|
27
|
+
ActiveRecord::Base.logger.tagged(Config.instance.log_tag) { select_rows(sql) }
|
28
|
+
else
|
29
|
+
select_rows(sql)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def select_rows(sql)
|
34
|
+
# This ensures the agent doesn't hold onto a DB connection any longer than necessary
|
35
|
+
ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "set"
|
4
|
+
require "rails_autoscale/metrics_collector"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
|
7
|
+
module RailsAutoscale
|
8
|
+
class JobMetricsCollector < MetricsCollector
|
9
|
+
include RailsAutoscale::Logger
|
10
|
+
|
11
|
+
# It's redundant to report these metrics from every dyno, so only report from the first one.
|
12
|
+
def self.collect?(config)
|
13
|
+
config.dyno.num == 1 && adapter_config.enabled
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.adapter_name
|
17
|
+
@_adapter_name ||= adapter_identifier.to_s.capitalize.gsub(/(?:_)(.)/i) { $1.upcase }
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.adapter_identifier
|
21
|
+
adapter_config.identifier
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.adapter_config
|
25
|
+
raise "Implement `self.adapter_config` in individual job metrics collectors."
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
super
|
30
|
+
|
31
|
+
log_msg = +"#{self.class.adapter_name} enabled"
|
32
|
+
log_msg << " with busy job tracking support" if track_busy_jobs?
|
33
|
+
logger.info log_msg
|
34
|
+
end
|
35
|
+
|
36
|
+
# Track the known queues so we can continue reporting on queues that don't
|
37
|
+
# have enqueued jobs at the time of reporting.
|
38
|
+
def queues
|
39
|
+
@queues ||= Set.new([])
|
40
|
+
end
|
41
|
+
|
42
|
+
def queues=(new_queues)
|
43
|
+
@queues = filter_queues(new_queues)
|
44
|
+
end
|
45
|
+
|
46
|
+
def clear_queues
|
47
|
+
@queues = nil
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def adapter_config
|
53
|
+
self.class.adapter_config
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter_queues(queues)
|
57
|
+
configured_queues = adapter_config.queues
|
58
|
+
|
59
|
+
if configured_queues.empty?
|
60
|
+
configured_filter = adapter_config.queue_filter
|
61
|
+
|
62
|
+
if configured_filter.respond_to?(:call)
|
63
|
+
queues = queues.select { |queue| configured_filter.call(queue) }
|
64
|
+
end
|
65
|
+
else
|
66
|
+
queues = configured_queues
|
67
|
+
end
|
68
|
+
|
69
|
+
queues = filter_max_queues(queues)
|
70
|
+
|
71
|
+
Set.new(queues)
|
72
|
+
end
|
73
|
+
|
74
|
+
# Collect up to the configured `max_queues`, skipping the rest.
|
75
|
+
# We sort queues by name length before making the cut-off, as a simple heuristic to keep the shorter ones
|
76
|
+
# and possibly ignore the longer ones, which are more likely to be dynamically generated for example.
|
77
|
+
def filter_max_queues(queues_to_collect)
|
78
|
+
queues_size = queues_to_collect.size
|
79
|
+
max_queues = adapter_config.max_queues
|
80
|
+
|
81
|
+
if queues_size > max_queues
|
82
|
+
logger.warn "#{self.class.adapter_name} metrics reporting only #{max_queues} queues max, skipping the rest (#{queues_size - max_queues})"
|
83
|
+
queues_to_collect.sort_by(&:length).first(max_queues)
|
84
|
+
else
|
85
|
+
queues_to_collect
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Sample log line for each collection, assuming `sidekiq` as the adapter identifier:
|
90
|
+
# `sidekiq-qt.default=10ms sidekiq-qd.default=3 sidekiq-busy.default=1`
|
91
|
+
def log_collection(metrics)
|
92
|
+
return if metrics.empty?
|
93
|
+
|
94
|
+
identifier = self.class.adapter_identifier
|
95
|
+
messages = metrics.map { |metric|
|
96
|
+
"#{identifier}-#{metric.identifier}.#{metric.queue_name}=#{metric.value}#{"ms" if metric.identifier == :qt}"
|
97
|
+
}
|
98
|
+
logger.debug messages.join(" ")
|
99
|
+
end
|
100
|
+
|
101
|
+
def track_busy_jobs?
|
102
|
+
adapter_config.track_busy_jobs
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/config"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
module Logger
|
8
|
+
def logger
|
9
|
+
@logger ||= LoggerProxy.new(Config.instance.logger, Config.instance.log_level)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class LoggerProxy < Struct.new(:logger, :log_level)
|
14
|
+
%w[ERROR WARN INFO DEBUG].each do |severity_name|
|
15
|
+
severity_level = ::Logger::Severity.const_get(severity_name)
|
16
|
+
|
17
|
+
define_method(severity_name.downcase) do |*messages|
|
18
|
+
if log_level.nil?
|
19
|
+
logger.add(severity_level) { tag(messages) }
|
20
|
+
elsif severity_level >= log_level
|
21
|
+
if severity_level >= logger.level
|
22
|
+
logger.add(severity_level) { tag(messages) }
|
23
|
+
else
|
24
|
+
logger.add(logger.level) { tag(messages, tag_level: severity_name) }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def tag(msgs, tag_level: nil)
|
33
|
+
tag = +"[#{Config.instance.log_tag}]"
|
34
|
+
tag << " [#{tag_level}]" if tag_level
|
35
|
+
msgs.map { |msg| "#{tag} #{msg}" }.join("\n")
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class Metric < Struct.new(:identifier, :value, :time, :queue_name)
|
5
|
+
# No queue_name is assumed to be a web request metric
|
6
|
+
# Metrics: qt = queue time (default), qd = queue depth, busy
|
7
|
+
def initialize(identifier, value, time, queue_name = nil)
|
8
|
+
super identifier, value.to_i, time.utc, queue_name
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "rails_autoscale/metric"
|
5
|
+
require "rails_autoscale/report"
|
6
|
+
|
7
|
+
module RailsAutoscale
|
8
|
+
class MetricsStore
|
9
|
+
include Singleton
|
10
|
+
|
11
|
+
attr_reader :metrics, :flushed_at
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@metrics = []
|
15
|
+
@flushed_at = Time.now
|
16
|
+
end
|
17
|
+
|
18
|
+
def push(identifier, value, time = Time.now, queue_name = nil)
|
19
|
+
# If it's been two minutes since clearing out the store, stop collecting metrics.
|
20
|
+
# There could be an issue with the reporter, and continuing to collect will consume linear memory.
|
21
|
+
return if @flushed_at && @flushed_at < Time.now - 120
|
22
|
+
|
23
|
+
@metrics << Metric.new(identifier, value, time, queue_name)
|
24
|
+
end
|
25
|
+
|
26
|
+
def flush
|
27
|
+
@flushed_at = Time.now
|
28
|
+
flushed_metrics = []
|
29
|
+
|
30
|
+
while (metric = @metrics.shift)
|
31
|
+
flushed_metrics << metric
|
32
|
+
end
|
33
|
+
|
34
|
+
flushed_metrics
|
35
|
+
end
|
36
|
+
|
37
|
+
def clear
|
38
|
+
@metrics.clear
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class Report
|
5
|
+
attr_reader :adapters, :config, :metrics
|
6
|
+
|
7
|
+
def initialize(adapters, config, metrics = [])
|
8
|
+
@adapters = adapters
|
9
|
+
@config = config
|
10
|
+
@metrics = metrics
|
11
|
+
end
|
12
|
+
|
13
|
+
def as_json
|
14
|
+
{
|
15
|
+
dyno: config.dyno,
|
16
|
+
pid: Process.pid,
|
17
|
+
config: config.as_json,
|
18
|
+
adapters: adapters.reduce({}) { |hash, adapter| hash.merge!(adapter.as_json) },
|
19
|
+
metrics: metrics.map { |metric|
|
20
|
+
[
|
21
|
+
metric.time.to_i,
|
22
|
+
metric.value,
|
23
|
+
metric.identifier,
|
24
|
+
metric.queue_name
|
25
|
+
]
|
26
|
+
}
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "rails_autoscale/config"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
require "rails_autoscale/adapter_api"
|
7
|
+
require "rails_autoscale/job_metrics_collector"
|
8
|
+
require "rails_autoscale/web_metrics_collector"
|
9
|
+
|
10
|
+
module RailsAutoscale
|
11
|
+
class Reporter
|
12
|
+
include Singleton
|
13
|
+
include Logger
|
14
|
+
|
15
|
+
def self.start(config = Config.instance, adapters = RailsAutoscale.adapters)
|
16
|
+
instance.start!(config, adapters) unless instance.started?
|
17
|
+
end
|
18
|
+
|
19
|
+
def start!(config, adapters)
|
20
|
+
@pid = Process.pid
|
21
|
+
|
22
|
+
if !config.api_base_url
|
23
|
+
logger.info "Reporter not started: RAILS_AUTOSCALE_URL is not set"
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
enabled_adapters = adapters.select { |adapter|
|
28
|
+
adapter.metrics_collector.nil? || adapter.metrics_collector.collect?(config)
|
29
|
+
}
|
30
|
+
metrics_collectors_classes = enabled_adapters.map(&:metrics_collector)
|
31
|
+
metrics_collectors_classes.compact!
|
32
|
+
|
33
|
+
if metrics_collectors_classes.empty?
|
34
|
+
logger.info "Reporter not started: no metrics need to be collected on this dyno"
|
35
|
+
return
|
36
|
+
end
|
37
|
+
|
38
|
+
adapters_msg = enabled_adapters.map(&:identifier).join(", ")
|
39
|
+
logger.info "Reporter starting, will report every #{config.report_interval_seconds} seconds or so. Adapters: [#{adapters_msg}]"
|
40
|
+
|
41
|
+
metrics_collectors = metrics_collectors_classes.map(&:new)
|
42
|
+
|
43
|
+
run_loop(config, metrics_collectors)
|
44
|
+
end
|
45
|
+
|
46
|
+
def run_loop(config, metrics_collectors)
|
47
|
+
@_thread = Thread.new do
|
48
|
+
loop do
|
49
|
+
run_metrics_collection(config, metrics_collectors)
|
50
|
+
|
51
|
+
# Stagger reporting to spread out reports from many processes
|
52
|
+
multiplier = 1 - (rand / 4) # between 0.75 and 1.0
|
53
|
+
sleep config.report_interval_seconds * multiplier
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def run_metrics_collection(config, metrics_collectors)
|
59
|
+
metrics = metrics_collectors.flat_map do |metric_collector|
|
60
|
+
log_exceptions { metric_collector.collect } || []
|
61
|
+
end
|
62
|
+
|
63
|
+
log_exceptions { report(config, metrics) }
|
64
|
+
end
|
65
|
+
|
66
|
+
def started?
|
67
|
+
@pid == Process.pid
|
68
|
+
end
|
69
|
+
|
70
|
+
def stop!
|
71
|
+
@_thread&.terminate
|
72
|
+
@_thread = nil
|
73
|
+
@pid = nil
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def report(config, metrics)
|
79
|
+
report = Report.new(RailsAutoscale.adapters, config, metrics)
|
80
|
+
logger.info "Reporting #{report.metrics.size} metrics"
|
81
|
+
result = AdapterApi.new(config).report_metrics(report.as_json)
|
82
|
+
|
83
|
+
case result
|
84
|
+
when AdapterApi::SuccessResponse
|
85
|
+
logger.debug "Reported successfully"
|
86
|
+
when AdapterApi::FailureResponse
|
87
|
+
logger.error "Reporter failed: #{result.failure_message}"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_exceptions
|
92
|
+
yield
|
93
|
+
rescue => ex
|
94
|
+
# Log the exception but swallow it to keep the thread running and processing reports.
|
95
|
+
# Note: Exceptions in threads other than the main thread will fail silently and terminate it.
|
96
|
+
# https://ruby-doc.org/core-3.1.0/Thread.html#class-Thread-label-Exception+handling
|
97
|
+
logger.error "Reporter error: #{ex.inspect}", *ex.backtrace
|
98
|
+
nil
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class RequestMetrics
|
5
|
+
attr_reader :request_id, :size, :network_time
|
6
|
+
|
7
|
+
def initialize(env, config = Config.instance)
|
8
|
+
@config = config
|
9
|
+
@request_id = env["HTTP_X_REQUEST_ID"]
|
10
|
+
@size = env["rack.input"].respond_to?(:size) ? env["rack.input"].size : 0
|
11
|
+
@network_time = env["puma.request_body_wait"].to_i
|
12
|
+
@request_start_header = env["HTTP_X_REQUEST_START"]
|
13
|
+
end
|
14
|
+
|
15
|
+
def ignore?
|
16
|
+
@config.ignore_large_requests? && @size > @config.max_request_size_bytes
|
17
|
+
end
|
18
|
+
|
19
|
+
def started_at
|
20
|
+
if @request_start_header
|
21
|
+
# Heroku sets the header as an integer, measured in milliseconds.
|
22
|
+
# If nginx is involved, it might be in seconds with fractional milliseconds,
|
23
|
+
# and it might be preceeded by "t=". We can all cases by removing non-digits
|
24
|
+
# and treating as milliseconds.
|
25
|
+
Time.at(@request_start_header.gsub(/\D/, "").to_i / 1000.0)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def queue_time(now = Time.now)
|
30
|
+
return if started_at.nil?
|
31
|
+
|
32
|
+
queue_time = ((now - started_at) * 1000).to_i
|
33
|
+
|
34
|
+
# Subtract the time Puma spent waiting on the request body, i.e. the network time. It's irrelevant to
|
35
|
+
# capacity-related queue time. Without this, slow clients and large request payloads will skew queue time.
|
36
|
+
queue_time -= network_time
|
37
|
+
|
38
|
+
# Safeguard against negative queue times (should not happen in practice)
|
39
|
+
queue_time > 0 ? queue_time : 0
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/metrics_store"
|
4
|
+
require "rails_autoscale/reporter"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
require "rails_autoscale/request_metrics"
|
7
|
+
|
8
|
+
module RailsAutoscale
|
9
|
+
class RequestMiddleware
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
def initialize(app)
|
13
|
+
@app = app
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(env)
|
17
|
+
request_metrics = RequestMetrics.new(env)
|
18
|
+
|
19
|
+
unless request_metrics.ignore?
|
20
|
+
queue_time = request_metrics.queue_time
|
21
|
+
network_time = request_metrics.network_time
|
22
|
+
end
|
23
|
+
|
24
|
+
Reporter.start
|
25
|
+
|
26
|
+
if queue_time
|
27
|
+
store = MetricsStore.instance
|
28
|
+
|
29
|
+
# NOTE: Expose queue time to the app
|
30
|
+
env["RailsAutoscale.queue_time"] = queue_time
|
31
|
+
store.push :qt, queue_time
|
32
|
+
|
33
|
+
unless network_time.zero?
|
34
|
+
env["RailsAutoscale.network_time"] = network_time
|
35
|
+
store.push :nt, network_time
|
36
|
+
end
|
37
|
+
|
38
|
+
logger.debug "Request queue_time=#{queue_time}ms network_time=#{network_time}ms request_id=#{request_metrics.request_id} size=#{request_metrics.size}"
|
39
|
+
end
|
40
|
+
|
41
|
+
@app.call(env)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/metrics_collector"
|
4
|
+
require "rails_autoscale/metrics_store"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
class WebMetricsCollector < MetricsCollector
|
8
|
+
def self.collect?(config)
|
9
|
+
config.dyno.name == "web"
|
10
|
+
end
|
11
|
+
|
12
|
+
def collect
|
13
|
+
MetricsStore.instance.flush
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require "rails_autoscale/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "rails-autoscale-core"
|
7
|
+
spec.version = RailsAutoscale::VERSION
|
8
|
+
spec.authors = ["Adam McCrea", "Carlos Antonio da Silva"]
|
9
|
+
spec.email = ["adam@adamlogic.com"]
|
10
|
+
|
11
|
+
spec.summary = "This gem works with the Rails Autoscale Heroku add-on to automatically scale your web and worker dynos."
|
12
|
+
spec.homepage = "https://railsautoscale.com"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.metadata = {
|
16
|
+
"homepage_uri" => "https://railsautoscale.com",
|
17
|
+
"bug_tracker_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems/issues",
|
18
|
+
"documentation_uri" => "https://railsautoscale.com/docs",
|
19
|
+
"changelog_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems/blob/main/CHANGELOG.md",
|
20
|
+
"source_code_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems"
|
21
|
+
}
|
22
|
+
|
23
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
24
|
+
spec.require_paths = ["lib"]
|
25
|
+
|
26
|
+
spec.required_ruby_version = ">= 2.6.0"
|
27
|
+
end
|
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rails-autoscale-core
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam McCrea
|
8
|
+
- Carlos Antonio da Silva
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2022-09-07 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description:
|
15
|
+
email:
|
16
|
+
- adam@adamlogic.com
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- Gemfile
|
22
|
+
- Gemfile.lock
|
23
|
+
- Rakefile
|
24
|
+
- lib/rails-autoscale-core.rb
|
25
|
+
- lib/rails_autoscale/adapter_api.rb
|
26
|
+
- lib/rails_autoscale/config.rb
|
27
|
+
- lib/rails_autoscale/job_metrics_collector.rb
|
28
|
+
- lib/rails_autoscale/job_metrics_collector/active_record_helper.rb
|
29
|
+
- lib/rails_autoscale/logger.rb
|
30
|
+
- lib/rails_autoscale/metric.rb
|
31
|
+
- lib/rails_autoscale/metrics_collector.rb
|
32
|
+
- lib/rails_autoscale/metrics_store.rb
|
33
|
+
- lib/rails_autoscale/report.rb
|
34
|
+
- lib/rails_autoscale/reporter.rb
|
35
|
+
- lib/rails_autoscale/request_metrics.rb
|
36
|
+
- lib/rails_autoscale/request_middleware.rb
|
37
|
+
- lib/rails_autoscale/version.rb
|
38
|
+
- lib/rails_autoscale/web_metrics_collector.rb
|
39
|
+
- rails-autoscale-core.gemspec
|
40
|
+
homepage: https://railsautoscale.com
|
41
|
+
licenses:
|
42
|
+
- MIT
|
43
|
+
metadata:
|
44
|
+
homepage_uri: https://railsautoscale.com
|
45
|
+
bug_tracker_uri: https://github.com/rails-autoscale/rails-autoscale-gems/issues
|
46
|
+
documentation_uri: https://railsautoscale.com/docs
|
47
|
+
changelog_uri: https://github.com/rails-autoscale/rails-autoscale-gems/blob/main/CHANGELOG.md
|
48
|
+
source_code_uri: https://github.com/rails-autoscale/rails-autoscale-gems
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 2.6.0
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubygems_version: 3.2.32
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale
|
68
|
+
your web and worker dynos.
|
69
|
+
test_files: []
|