rails-autoscale-core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +36 -0
- data/Rakefile +12 -0
- data/lib/rails-autoscale-core.rb +40 -0
- data/lib/rails_autoscale/adapter_api.rb +58 -0
- data/lib/rails_autoscale/config.rb +114 -0
- data/lib/rails_autoscale/job_metrics_collector/active_record_helper.rb +39 -0
- data/lib/rails_autoscale/job_metrics_collector.rb +105 -0
- data/lib/rails_autoscale/logger.rb +38 -0
- data/lib/rails_autoscale/metric.rb +11 -0
- data/lib/rails_autoscale/metrics_collector.rb +13 -0
- data/lib/rails_autoscale/metrics_store.rb +41 -0
- data/lib/rails_autoscale/report.rb +30 -0
- data/lib/rails_autoscale/reporter.rb +101 -0
- data/lib/rails_autoscale/request_metrics.rb +42 -0
- data/lib/rails_autoscale/request_middleware.rb +44 -0
- data/lib/rails_autoscale/version.rb +5 -0
- data/lib/rails_autoscale/web_metrics_collector.rb +16 -0
- data/rails-autoscale-core.gemspec +27 -0
- metadata +69 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 957a594b6d2686be815ed71d6c84ffcc92e14a14f5a9035fb12f92d6f979a400
|
4
|
+
data.tar.gz: b8cb713fb2731acbc11724097425d497c1f48e5bacaeff5cc41410a8b031f0a3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4b79400a71ac5f68401be26efa1bcc9e723b37d55c23802fb0ec30a9d1963bc892ac7b0c8067f11cddc802ffbd3dffd2cc80d7895ab7a1be89d0a7aebcff5232
|
7
|
+
data.tar.gz: f8940544874532959ae2ab3d5405c7087023a742a4f2a365d6ae4a2b15a602e6a41f9c6862d3171b55e4000e7c08008d37781f77531c0c492ea69c926a0377c1
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
rails-autoscale-core (1.0.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
addressable (2.8.0)
|
10
|
+
public_suffix (>= 2.0.2, < 5.0)
|
11
|
+
crack (0.4.5)
|
12
|
+
rexml
|
13
|
+
hashdiff (1.0.1)
|
14
|
+
minitest (5.15.0)
|
15
|
+
public_suffix (4.0.6)
|
16
|
+
rake (13.0.6)
|
17
|
+
rexml (3.2.5)
|
18
|
+
webmock (3.14.0)
|
19
|
+
addressable (>= 2.8.0)
|
20
|
+
crack (>= 0.3.2)
|
21
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
22
|
+
|
23
|
+
PLATFORMS
|
24
|
+
arm64-darwin-20
|
25
|
+
arm64-darwin-21
|
26
|
+
x86_64-darwin-21
|
27
|
+
x86_64-linux
|
28
|
+
|
29
|
+
DEPENDENCIES
|
30
|
+
minitest
|
31
|
+
rails-autoscale-core!
|
32
|
+
rake (>= 12.3.3)
|
33
|
+
webmock
|
34
|
+
|
35
|
+
BUNDLED WITH
|
36
|
+
2.3.9
|
data/Rakefile
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/config"
|
4
|
+
require "rails_autoscale/version"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
# Allows configuring Rails Autoscale through a block, usually defined during application initialization.
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
#
|
11
|
+
# RailsAutoscale.configure do |config|
|
12
|
+
# config.logger = MyLogger.new
|
13
|
+
# end
|
14
|
+
def self.configure
|
15
|
+
yield Config.instance
|
16
|
+
end
|
17
|
+
|
18
|
+
@adapters = []
|
19
|
+
class << self
|
20
|
+
attr_reader :adapters
|
21
|
+
end
|
22
|
+
|
23
|
+
Adapter = Struct.new(:identifier, :adapter_info, :metrics_collector) do
|
24
|
+
def as_json
|
25
|
+
{identifier => adapter_info}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.add_adapter(identifier, adapter_info, metrics_collector: nil, expose_config: nil)
|
30
|
+
Config.expose_adapter_config(expose_config) if expose_config
|
31
|
+
@adapters << Adapter.new(identifier, adapter_info, metrics_collector)
|
32
|
+
end
|
33
|
+
|
34
|
+
add_adapter :"rails-autoscale-core", {
|
35
|
+
adapter_version: VERSION,
|
36
|
+
language_version: RUBY_VERSION
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
Judoscale = RailsAutoscale
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "net/http"
|
4
|
+
require "uri"
|
5
|
+
require "json"
|
6
|
+
require "rails_autoscale/logger"
|
7
|
+
|
8
|
+
module RailsAutoscale
|
9
|
+
class AdapterApi
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
SUCCESS = "success"
|
13
|
+
|
14
|
+
def initialize(config)
|
15
|
+
@config = config
|
16
|
+
end
|
17
|
+
|
18
|
+
def report_metrics(report_json)
|
19
|
+
post_json "/v3/reports", report_json
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def post_json(path, data)
|
25
|
+
headers = {"Content-Type" => "application/json"}
|
26
|
+
post_raw path: path, body: JSON.dump(data), headers: headers
|
27
|
+
end
|
28
|
+
|
29
|
+
def post_raw(options)
|
30
|
+
uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
|
31
|
+
ssl = uri.scheme == "https"
|
32
|
+
|
33
|
+
response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
|
34
|
+
request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
|
35
|
+
request.body = options.fetch(:body)
|
36
|
+
|
37
|
+
logger.debug "Posting #{request.body.size} bytes to #{uri}"
|
38
|
+
http.request(request)
|
39
|
+
end
|
40
|
+
|
41
|
+
case response.code.to_i
|
42
|
+
when 200...300 then SuccessResponse.new(response.body)
|
43
|
+
else FailureResponse.new([response.code, response.message].join(" - "))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class SuccessResponse < Struct.new(:body)
|
48
|
+
def data
|
49
|
+
JSON.parse(body)
|
50
|
+
rescue TypeError
|
51
|
+
{}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class FailureResponse < Struct.new(:failure_message)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
class Config
|
8
|
+
class Dyno
|
9
|
+
attr_reader :name, :num
|
10
|
+
|
11
|
+
def initialize(dyno_string)
|
12
|
+
@name, @num = dyno_string.to_s.split(".")
|
13
|
+
@num = @num.to_i
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_s
|
17
|
+
"#{name}.#{num}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class JobAdapterConfig
|
22
|
+
UUID_REGEXP = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/
|
23
|
+
DEFAULT_QUEUE_FILTER = ->(queue_name) { !UUID_REGEXP.match?(queue_name) }
|
24
|
+
|
25
|
+
attr_accessor :identifier, :enabled, :max_queues, :queues, :queue_filter, :track_busy_jobs
|
26
|
+
|
27
|
+
def initialize(identifier)
|
28
|
+
@identifier = identifier
|
29
|
+
reset
|
30
|
+
end
|
31
|
+
|
32
|
+
def reset
|
33
|
+
@enabled = true
|
34
|
+
@max_queues = 20
|
35
|
+
@queues = []
|
36
|
+
@queue_filter = DEFAULT_QUEUE_FILTER
|
37
|
+
@track_busy_jobs = false
|
38
|
+
end
|
39
|
+
|
40
|
+
def as_json
|
41
|
+
{
|
42
|
+
identifier => {
|
43
|
+
max_queues: max_queues,
|
44
|
+
queues: queues,
|
45
|
+
queue_filter: queue_filter != DEFAULT_QUEUE_FILTER,
|
46
|
+
track_busy_jobs: track_busy_jobs
|
47
|
+
}
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
include Singleton
|
53
|
+
|
54
|
+
@adapter_configs = []
|
55
|
+
class << self
|
56
|
+
attr_reader :adapter_configs
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.expose_adapter_config(config_instance)
|
60
|
+
adapter_configs << config_instance
|
61
|
+
|
62
|
+
define_method(config_instance.identifier) do
|
63
|
+
config_instance
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
attr_accessor :api_base_url, :report_interval_seconds, :max_request_size_bytes, :logger, :log_tag
|
68
|
+
attr_reader :dyno, :log_level
|
69
|
+
|
70
|
+
def initialize
|
71
|
+
reset
|
72
|
+
end
|
73
|
+
|
74
|
+
def reset
|
75
|
+
# Allow the API URL to be configured - needed for testing.
|
76
|
+
@api_base_url = ENV["RAILS_AUTOSCALE_URL"] || ENV["JUDOSCALE_URL"]
|
77
|
+
@log_tag = ENV["JUDOSCALE_URL"] ? "Judoscale" : "RailsAutoscale"
|
78
|
+
self.dyno = ENV["DYNO"]
|
79
|
+
@max_request_size_bytes = 100_000 # ignore request payloads over 100k since they skew the queue times
|
80
|
+
@report_interval_seconds = 10
|
81
|
+
self.log_level = ENV["RAILS_AUTOSCALE_LOG_LEVEL"] || ENV["JUDOSCALE_LOG_LEVEL"]
|
82
|
+
@logger = ::Logger.new($stdout)
|
83
|
+
|
84
|
+
self.class.adapter_configs.each(&:reset)
|
85
|
+
end
|
86
|
+
|
87
|
+
def dyno=(dyno_string)
|
88
|
+
@dyno = Dyno.new(dyno_string)
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_level=(new_level)
|
92
|
+
@log_level = new_level ? ::Logger::Severity.const_get(new_level.to_s.upcase) : nil
|
93
|
+
end
|
94
|
+
|
95
|
+
def as_json
|
96
|
+
adapter_configs_json = self.class.adapter_configs.reduce({}) { |hash, config| hash.merge!(config.as_json) }
|
97
|
+
|
98
|
+
{
|
99
|
+
log_level: log_level,
|
100
|
+
logger: logger.class.name,
|
101
|
+
report_interval_seconds: report_interval_seconds,
|
102
|
+
max_request_size_bytes: max_request_size_bytes
|
103
|
+
}.merge!(adapter_configs_json)
|
104
|
+
end
|
105
|
+
|
106
|
+
def to_s
|
107
|
+
"#{@dyno}##{Process.pid}"
|
108
|
+
end
|
109
|
+
|
110
|
+
def ignore_large_requests?
|
111
|
+
@max_request_size_bytes
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class JobMetricsCollector
|
5
|
+
module ActiveRecordHelper
|
6
|
+
# Cleanup any whitespace characters (including new lines) from the SQL for simpler logging.
|
7
|
+
# Reference: ActiveSupport's `squish!` method. https://api.rubyonrails.org/classes/String.html#method-i-squish
|
8
|
+
def self.cleanse_sql(sql)
|
9
|
+
sql = sql.dup
|
10
|
+
sql.gsub!(/[[:space:]]+/, " ")
|
11
|
+
sql.strip!
|
12
|
+
sql
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def select_rows_silently(sql)
|
18
|
+
if Config.instance.log_level && ::ActiveRecord::Base.logger.respond_to?(:silence)
|
19
|
+
::ActiveRecord::Base.logger.silence(Config.instance.log_level) { select_rows_tagged(sql) }
|
20
|
+
else
|
21
|
+
select_rows_tagged(sql)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def select_rows_tagged(sql)
|
26
|
+
if ActiveRecord::Base.logger.respond_to?(:tagged)
|
27
|
+
ActiveRecord::Base.logger.tagged(Config.instance.log_tag) { select_rows(sql) }
|
28
|
+
else
|
29
|
+
select_rows(sql)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def select_rows(sql)
|
34
|
+
# This ensures the agent doesn't hold onto a DB connection any longer than necessary
|
35
|
+
ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "set"
|
4
|
+
require "rails_autoscale/metrics_collector"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
|
7
|
+
module RailsAutoscale
|
8
|
+
class JobMetricsCollector < MetricsCollector
|
9
|
+
include RailsAutoscale::Logger
|
10
|
+
|
11
|
+
# It's redundant to report these metrics from every dyno, so only report from the first one.
|
12
|
+
def self.collect?(config)
|
13
|
+
config.dyno.num == 1 && adapter_config.enabled
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.adapter_name
|
17
|
+
@_adapter_name ||= adapter_identifier.to_s.capitalize.gsub(/(?:_)(.)/i) { $1.upcase }
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.adapter_identifier
|
21
|
+
adapter_config.identifier
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.adapter_config
|
25
|
+
raise "Implement `self.adapter_config` in individual job metrics collectors."
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
super
|
30
|
+
|
31
|
+
log_msg = +"#{self.class.adapter_name} enabled"
|
32
|
+
log_msg << " with busy job tracking support" if track_busy_jobs?
|
33
|
+
logger.info log_msg
|
34
|
+
end
|
35
|
+
|
36
|
+
# Track the known queues so we can continue reporting on queues that don't
|
37
|
+
# have enqueued jobs at the time of reporting.
|
38
|
+
def queues
|
39
|
+
@queues ||= Set.new([])
|
40
|
+
end
|
41
|
+
|
42
|
+
def queues=(new_queues)
|
43
|
+
@queues = filter_queues(new_queues)
|
44
|
+
end
|
45
|
+
|
46
|
+
def clear_queues
|
47
|
+
@queues = nil
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def adapter_config
|
53
|
+
self.class.adapter_config
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter_queues(queues)
|
57
|
+
configured_queues = adapter_config.queues
|
58
|
+
|
59
|
+
if configured_queues.empty?
|
60
|
+
configured_filter = adapter_config.queue_filter
|
61
|
+
|
62
|
+
if configured_filter.respond_to?(:call)
|
63
|
+
queues = queues.select { |queue| configured_filter.call(queue) }
|
64
|
+
end
|
65
|
+
else
|
66
|
+
queues = configured_queues
|
67
|
+
end
|
68
|
+
|
69
|
+
queues = filter_max_queues(queues)
|
70
|
+
|
71
|
+
Set.new(queues)
|
72
|
+
end
|
73
|
+
|
74
|
+
# Collect up to the configured `max_queues`, skipping the rest.
|
75
|
+
# We sort queues by name length before making the cut-off, as a simple heuristic to keep the shorter ones
|
76
|
+
# and possibly ignore the longer ones, which are more likely to be dynamically generated for example.
|
77
|
+
def filter_max_queues(queues_to_collect)
|
78
|
+
queues_size = queues_to_collect.size
|
79
|
+
max_queues = adapter_config.max_queues
|
80
|
+
|
81
|
+
if queues_size > max_queues
|
82
|
+
logger.warn "#{self.class.adapter_name} metrics reporting only #{max_queues} queues max, skipping the rest (#{queues_size - max_queues})"
|
83
|
+
queues_to_collect.sort_by(&:length).first(max_queues)
|
84
|
+
else
|
85
|
+
queues_to_collect
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Sample log line for each collection, assuming `sidekiq` as the adapter identifier:
|
90
|
+
# `sidekiq-qt.default=10ms sidekiq-qd.default=3 sidekiq-busy.default=1`
|
91
|
+
def log_collection(metrics)
|
92
|
+
return if metrics.empty?
|
93
|
+
|
94
|
+
identifier = self.class.adapter_identifier
|
95
|
+
messages = metrics.map { |metric|
|
96
|
+
"#{identifier}-#{metric.identifier}.#{metric.queue_name}=#{metric.value}#{"ms" if metric.identifier == :qt}"
|
97
|
+
}
|
98
|
+
logger.debug messages.join(" ")
|
99
|
+
end
|
100
|
+
|
101
|
+
def track_busy_jobs?
|
102
|
+
adapter_config.track_busy_jobs
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/config"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
module Logger
|
8
|
+
def logger
|
9
|
+
@logger ||= LoggerProxy.new(Config.instance.logger, Config.instance.log_level)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class LoggerProxy < Struct.new(:logger, :log_level)
|
14
|
+
%w[ERROR WARN INFO DEBUG].each do |severity_name|
|
15
|
+
severity_level = ::Logger::Severity.const_get(severity_name)
|
16
|
+
|
17
|
+
define_method(severity_name.downcase) do |*messages|
|
18
|
+
if log_level.nil?
|
19
|
+
logger.add(severity_level) { tag(messages) }
|
20
|
+
elsif severity_level >= log_level
|
21
|
+
if severity_level >= logger.level
|
22
|
+
logger.add(severity_level) { tag(messages) }
|
23
|
+
else
|
24
|
+
logger.add(logger.level) { tag(messages, tag_level: severity_name) }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def tag(msgs, tag_level: nil)
|
33
|
+
tag = +"[#{Config.instance.log_tag}]"
|
34
|
+
tag << " [#{tag_level}]" if tag_level
|
35
|
+
msgs.map { |msg| "#{tag} #{msg}" }.join("\n")
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class Metric < Struct.new(:identifier, :value, :time, :queue_name)
|
5
|
+
# No queue_name is assumed to be a web request metric
|
6
|
+
# Metrics: qt = queue time (default), qd = queue depth, busy
|
7
|
+
def initialize(identifier, value, time, queue_name = nil)
|
8
|
+
super identifier, value.to_i, time.utc, queue_name
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "rails_autoscale/metric"
|
5
|
+
require "rails_autoscale/report"
|
6
|
+
|
7
|
+
module RailsAutoscale
|
8
|
+
class MetricsStore
|
9
|
+
include Singleton
|
10
|
+
|
11
|
+
attr_reader :metrics, :flushed_at
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@metrics = []
|
15
|
+
@flushed_at = Time.now
|
16
|
+
end
|
17
|
+
|
18
|
+
def push(identifier, value, time = Time.now, queue_name = nil)
|
19
|
+
# If it's been two minutes since clearing out the store, stop collecting metrics.
|
20
|
+
# There could be an issue with the reporter, and continuing to collect will consume linear memory.
|
21
|
+
return if @flushed_at && @flushed_at < Time.now - 120
|
22
|
+
|
23
|
+
@metrics << Metric.new(identifier, value, time, queue_name)
|
24
|
+
end
|
25
|
+
|
26
|
+
def flush
|
27
|
+
@flushed_at = Time.now
|
28
|
+
flushed_metrics = []
|
29
|
+
|
30
|
+
while (metric = @metrics.shift)
|
31
|
+
flushed_metrics << metric
|
32
|
+
end
|
33
|
+
|
34
|
+
flushed_metrics
|
35
|
+
end
|
36
|
+
|
37
|
+
def clear
|
38
|
+
@metrics.clear
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class Report
|
5
|
+
attr_reader :adapters, :config, :metrics
|
6
|
+
|
7
|
+
def initialize(adapters, config, metrics = [])
|
8
|
+
@adapters = adapters
|
9
|
+
@config = config
|
10
|
+
@metrics = metrics
|
11
|
+
end
|
12
|
+
|
13
|
+
def as_json
|
14
|
+
{
|
15
|
+
dyno: config.dyno,
|
16
|
+
pid: Process.pid,
|
17
|
+
config: config.as_json,
|
18
|
+
adapters: adapters.reduce({}) { |hash, adapter| hash.merge!(adapter.as_json) },
|
19
|
+
metrics: metrics.map { |metric|
|
20
|
+
[
|
21
|
+
metric.time.to_i,
|
22
|
+
metric.value,
|
23
|
+
metric.identifier,
|
24
|
+
metric.queue_name
|
25
|
+
]
|
26
|
+
}
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
require "rails_autoscale/config"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
require "rails_autoscale/adapter_api"
|
7
|
+
require "rails_autoscale/job_metrics_collector"
|
8
|
+
require "rails_autoscale/web_metrics_collector"
|
9
|
+
|
10
|
+
module RailsAutoscale
|
11
|
+
class Reporter
|
12
|
+
include Singleton
|
13
|
+
include Logger
|
14
|
+
|
15
|
+
def self.start(config = Config.instance, adapters = RailsAutoscale.adapters)
|
16
|
+
instance.start!(config, adapters) unless instance.started?
|
17
|
+
end
|
18
|
+
|
19
|
+
def start!(config, adapters)
|
20
|
+
@pid = Process.pid
|
21
|
+
|
22
|
+
if !config.api_base_url
|
23
|
+
logger.info "Reporter not started: RAILS_AUTOSCALE_URL is not set"
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
enabled_adapters = adapters.select { |adapter|
|
28
|
+
adapter.metrics_collector.nil? || adapter.metrics_collector.collect?(config)
|
29
|
+
}
|
30
|
+
metrics_collectors_classes = enabled_adapters.map(&:metrics_collector)
|
31
|
+
metrics_collectors_classes.compact!
|
32
|
+
|
33
|
+
if metrics_collectors_classes.empty?
|
34
|
+
logger.info "Reporter not started: no metrics need to be collected on this dyno"
|
35
|
+
return
|
36
|
+
end
|
37
|
+
|
38
|
+
adapters_msg = enabled_adapters.map(&:identifier).join(", ")
|
39
|
+
logger.info "Reporter starting, will report every #{config.report_interval_seconds} seconds or so. Adapters: [#{adapters_msg}]"
|
40
|
+
|
41
|
+
metrics_collectors = metrics_collectors_classes.map(&:new)
|
42
|
+
|
43
|
+
run_loop(config, metrics_collectors)
|
44
|
+
end
|
45
|
+
|
46
|
+
def run_loop(config, metrics_collectors)
|
47
|
+
@_thread = Thread.new do
|
48
|
+
loop do
|
49
|
+
run_metrics_collection(config, metrics_collectors)
|
50
|
+
|
51
|
+
# Stagger reporting to spread out reports from many processes
|
52
|
+
multiplier = 1 - (rand / 4) # between 0.75 and 1.0
|
53
|
+
sleep config.report_interval_seconds * multiplier
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def run_metrics_collection(config, metrics_collectors)
|
59
|
+
metrics = metrics_collectors.flat_map do |metric_collector|
|
60
|
+
log_exceptions { metric_collector.collect } || []
|
61
|
+
end
|
62
|
+
|
63
|
+
log_exceptions { report(config, metrics) }
|
64
|
+
end
|
65
|
+
|
66
|
+
def started?
|
67
|
+
@pid == Process.pid
|
68
|
+
end
|
69
|
+
|
70
|
+
def stop!
|
71
|
+
@_thread&.terminate
|
72
|
+
@_thread = nil
|
73
|
+
@pid = nil
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def report(config, metrics)
|
79
|
+
report = Report.new(RailsAutoscale.adapters, config, metrics)
|
80
|
+
logger.info "Reporting #{report.metrics.size} metrics"
|
81
|
+
result = AdapterApi.new(config).report_metrics(report.as_json)
|
82
|
+
|
83
|
+
case result
|
84
|
+
when AdapterApi::SuccessResponse
|
85
|
+
logger.debug "Reported successfully"
|
86
|
+
when AdapterApi::FailureResponse
|
87
|
+
logger.error "Reporter failed: #{result.failure_message}"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_exceptions
|
92
|
+
yield
|
93
|
+
rescue => ex
|
94
|
+
# Log the exception but swallow it to keep the thread running and processing reports.
|
95
|
+
# Note: Exceptions in threads other than the main thread will fail silently and terminate it.
|
96
|
+
# https://ruby-doc.org/core-3.1.0/Thread.html#class-Thread-label-Exception+handling
|
97
|
+
logger.error "Reporter error: #{ex.inspect}", *ex.backtrace
|
98
|
+
nil
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailsAutoscale
|
4
|
+
class RequestMetrics
|
5
|
+
attr_reader :request_id, :size, :network_time
|
6
|
+
|
7
|
+
def initialize(env, config = Config.instance)
|
8
|
+
@config = config
|
9
|
+
@request_id = env["HTTP_X_REQUEST_ID"]
|
10
|
+
@size = env["rack.input"].respond_to?(:size) ? env["rack.input"].size : 0
|
11
|
+
@network_time = env["puma.request_body_wait"].to_i
|
12
|
+
@request_start_header = env["HTTP_X_REQUEST_START"]
|
13
|
+
end
|
14
|
+
|
15
|
+
def ignore?
|
16
|
+
@config.ignore_large_requests? && @size > @config.max_request_size_bytes
|
17
|
+
end
|
18
|
+
|
19
|
+
def started_at
|
20
|
+
if @request_start_header
|
21
|
+
# Heroku sets the header as an integer, measured in milliseconds.
|
22
|
+
# If nginx is involved, it might be in seconds with fractional milliseconds,
|
23
|
+
# and it might be preceeded by "t=". We can all cases by removing non-digits
|
24
|
+
# and treating as milliseconds.
|
25
|
+
Time.at(@request_start_header.gsub(/\D/, "").to_i / 1000.0)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def queue_time(now = Time.now)
|
30
|
+
return if started_at.nil?
|
31
|
+
|
32
|
+
queue_time = ((now - started_at) * 1000).to_i
|
33
|
+
|
34
|
+
# Subtract the time Puma spent waiting on the request body, i.e. the network time. It's irrelevant to
|
35
|
+
# capacity-related queue time. Without this, slow clients and large request payloads will skew queue time.
|
36
|
+
queue_time -= network_time
|
37
|
+
|
38
|
+
# Safeguard against negative queue times (should not happen in practice)
|
39
|
+
queue_time > 0 ? queue_time : 0
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/metrics_store"
|
4
|
+
require "rails_autoscale/reporter"
|
5
|
+
require "rails_autoscale/logger"
|
6
|
+
require "rails_autoscale/request_metrics"
|
7
|
+
|
8
|
+
module RailsAutoscale
|
9
|
+
class RequestMiddleware
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
def initialize(app)
|
13
|
+
@app = app
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(env)
|
17
|
+
request_metrics = RequestMetrics.new(env)
|
18
|
+
|
19
|
+
unless request_metrics.ignore?
|
20
|
+
queue_time = request_metrics.queue_time
|
21
|
+
network_time = request_metrics.network_time
|
22
|
+
end
|
23
|
+
|
24
|
+
Reporter.start
|
25
|
+
|
26
|
+
if queue_time
|
27
|
+
store = MetricsStore.instance
|
28
|
+
|
29
|
+
# NOTE: Expose queue time to the app
|
30
|
+
env["RailsAutoscale.queue_time"] = queue_time
|
31
|
+
store.push :qt, queue_time
|
32
|
+
|
33
|
+
unless network_time.zero?
|
34
|
+
env["RailsAutoscale.network_time"] = network_time
|
35
|
+
store.push :nt, network_time
|
36
|
+
end
|
37
|
+
|
38
|
+
logger.debug "Request queue_time=#{queue_time}ms network_time=#{network_time}ms request_id=#{request_metrics.request_id} size=#{request_metrics.size}"
|
39
|
+
end
|
40
|
+
|
41
|
+
@app.call(env)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_autoscale/metrics_collector"
|
4
|
+
require "rails_autoscale/metrics_store"
|
5
|
+
|
6
|
+
module RailsAutoscale
|
7
|
+
class WebMetricsCollector < MetricsCollector
|
8
|
+
def self.collect?(config)
|
9
|
+
config.dyno.name == "web"
|
10
|
+
end
|
11
|
+
|
12
|
+
def collect
|
13
|
+
MetricsStore.instance.flush
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require "rails_autoscale/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "rails-autoscale-core"
|
7
|
+
spec.version = RailsAutoscale::VERSION
|
8
|
+
spec.authors = ["Adam McCrea", "Carlos Antonio da Silva"]
|
9
|
+
spec.email = ["adam@adamlogic.com"]
|
10
|
+
|
11
|
+
spec.summary = "This gem works with the Rails Autoscale Heroku add-on to automatically scale your web and worker dynos."
|
12
|
+
spec.homepage = "https://railsautoscale.com"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.metadata = {
|
16
|
+
"homepage_uri" => "https://railsautoscale.com",
|
17
|
+
"bug_tracker_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems/issues",
|
18
|
+
"documentation_uri" => "https://railsautoscale.com/docs",
|
19
|
+
"changelog_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems/blob/main/CHANGELOG.md",
|
20
|
+
"source_code_uri" => "https://github.com/rails-autoscale/rails-autoscale-gems"
|
21
|
+
}
|
22
|
+
|
23
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
24
|
+
spec.require_paths = ["lib"]
|
25
|
+
|
26
|
+
spec.required_ruby_version = ">= 2.6.0"
|
27
|
+
end
|
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rails-autoscale-core
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam McCrea
|
8
|
+
- Carlos Antonio da Silva
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2022-09-07 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description:
|
15
|
+
email:
|
16
|
+
- adam@adamlogic.com
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- Gemfile
|
22
|
+
- Gemfile.lock
|
23
|
+
- Rakefile
|
24
|
+
- lib/rails-autoscale-core.rb
|
25
|
+
- lib/rails_autoscale/adapter_api.rb
|
26
|
+
- lib/rails_autoscale/config.rb
|
27
|
+
- lib/rails_autoscale/job_metrics_collector.rb
|
28
|
+
- lib/rails_autoscale/job_metrics_collector/active_record_helper.rb
|
29
|
+
- lib/rails_autoscale/logger.rb
|
30
|
+
- lib/rails_autoscale/metric.rb
|
31
|
+
- lib/rails_autoscale/metrics_collector.rb
|
32
|
+
- lib/rails_autoscale/metrics_store.rb
|
33
|
+
- lib/rails_autoscale/report.rb
|
34
|
+
- lib/rails_autoscale/reporter.rb
|
35
|
+
- lib/rails_autoscale/request_metrics.rb
|
36
|
+
- lib/rails_autoscale/request_middleware.rb
|
37
|
+
- lib/rails_autoscale/version.rb
|
38
|
+
- lib/rails_autoscale/web_metrics_collector.rb
|
39
|
+
- rails-autoscale-core.gemspec
|
40
|
+
homepage: https://railsautoscale.com
|
41
|
+
licenses:
|
42
|
+
- MIT
|
43
|
+
metadata:
|
44
|
+
homepage_uri: https://railsautoscale.com
|
45
|
+
bug_tracker_uri: https://github.com/rails-autoscale/rails-autoscale-gems/issues
|
46
|
+
documentation_uri: https://railsautoscale.com/docs
|
47
|
+
changelog_uri: https://github.com/rails-autoscale/rails-autoscale-gems/blob/main/CHANGELOG.md
|
48
|
+
source_code_uri: https://github.com/rails-autoscale/rails-autoscale-gems
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 2.6.0
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubygems_version: 3.2.32
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale
|
68
|
+
your web and worker dynos.
|
69
|
+
test_files: []
|