heroku-vector 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +37 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +95 -0
- data/README.md +171 -0
- data/Rakefile +10 -0
- data/bin/heroku_vector +91 -0
- data/config.rb.example +28 -0
- data/diagrams/Architecture.pptx +0 -0
- data/diagrams/Architecture/Slide1.png +0 -0
- data/examples/upstart_conf +10 -0
- data/heroku-vector.gemspec +38 -0
- data/lib/heroku_vector.rb +133 -0
- data/lib/heroku_vector/dyno_scaler.rb +133 -0
- data/lib/heroku_vector/engine/heroku.rb +63 -0
- data/lib/heroku_vector/helper.rb +23 -0
- data/lib/heroku_vector/process_manager.rb +61 -0
- data/lib/heroku_vector/sampler.rb +50 -0
- data/lib/heroku_vector/source/new_relic.rb +50 -0
- data/lib/heroku_vector/source/sidekiq.rb +49 -0
- data/lib/heroku_vector/version.rb +3 -0
- data/lib/heroku_vector/worker.rb +49 -0
- data/test/fixtures/vcr_cassettes/heroku_dyno_types.yml +69 -0
- data/test/fixtures/vcr_cassettes/heroku_scale_dynos.yml +65 -0
- data/test/fixtures/vcr_cassettes/newrelic_account.yml +78 -0
- data/test/fixtures/vcr_cassettes/newrelic_app.yml +134 -0
- data/test/fixtures/vcr_cassettes/newrelic_threshold_values.yml +192 -0
- data/test/heroku_vector/dyno_scaler_test.rb +240 -0
- data/test/heroku_vector/engine/heroku_test.rb +91 -0
- data/test/heroku_vector/sampler_test.rb +61 -0
- data/test/heroku_vector/source/new_relic_test.rb +77 -0
- data/test/heroku_vector/source/sidekiq_test.rb +78 -0
- data/test/test_helper.rb +40 -0
- metadata +313 -0
Binary file
|
Binary file
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# Example Upstart config to run heroku_vector on Linux
|
2
|
+
# eg: /etc/init/heroku-vector.conf
|
3
|
+
|
4
|
+
respawn
|
5
|
+
respawn limit 15 5
|
6
|
+
|
7
|
+
start on runlevel [2345]
|
8
|
+
stop on runlevel [06]
|
9
|
+
|
10
|
+
exec /usr/local/bin/heroku_vector -c /path/to/your/config.rb -l /var/log/heroku_vector.log
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'heroku_vector/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "heroku-vector"
|
8
|
+
spec.version = HerokuVector::VERSION
|
9
|
+
spec.authors = ["Winfield Peterson"]
|
10
|
+
spec.email = ["winfield.peterson@gmail.com"]
|
11
|
+
spec.summary = %q{Sampling Auto-scaler for Heroku dynos}
|
12
|
+
spec.description = %q{Linearly scale Heroku dyno counts based on sampled metrics}
|
13
|
+
spec.homepage = "https://github.com/wpeterson/heroku-vector"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "activeresource"
|
22
|
+
spec.add_dependency "heroku-api"
|
23
|
+
spec.add_dependency "newrelic_api"
|
24
|
+
spec.add_dependency "redis"
|
25
|
+
spec.add_dependency "redis-namespace"
|
26
|
+
spec.add_dependency "sidekiq"
|
27
|
+
spec.add_dependency "dotenv"
|
28
|
+
spec.add_dependency "eventmachine"
|
29
|
+
|
30
|
+
spec.add_development_dependency "minitest"
|
31
|
+
spec.add_development_dependency "mocha"
|
32
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
33
|
+
spec.add_development_dependency "rake"
|
34
|
+
spec.add_development_dependency "timecop"
|
35
|
+
spec.add_development_dependency "webmock"
|
36
|
+
spec.add_development_dependency "vcr"
|
37
|
+
spec.add_development_dependency "pry"
|
38
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
module HerokuVector
|
2
|
+
module Engine
|
3
|
+
end
|
4
|
+
module Source
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
8
|
+
require 'logger'
|
9
|
+
|
10
|
+
require 'heroku_vector/helper'
|
11
|
+
|
12
|
+
require 'heroku_vector/dyno_scaler'
|
13
|
+
require 'heroku_vector/process_manager'
|
14
|
+
require 'heroku_vector/sampler'
|
15
|
+
require 'heroku_vector/worker'
|
16
|
+
require 'heroku_vector/version'
|
17
|
+
require 'heroku_vector/engine/heroku'
|
18
|
+
require 'heroku_vector/source/new_relic'
|
19
|
+
require 'heroku_vector/source/sidekiq'
|
20
|
+
|
21
|
+
module HerokuVector
|
22
|
+
class << self
|
23
|
+
|
24
|
+
def default_logger
|
25
|
+
return options[:logger] if options[:logger]
|
26
|
+
|
27
|
+
logger = Logger.new(STDOUT)
|
28
|
+
logger.level = options[:log_level]
|
29
|
+
logger.formatter = proc do |severity, time, program_name, message|
|
30
|
+
"#{time.utc.iso8601(3)} #{severity}: #{message}\n"
|
31
|
+
end
|
32
|
+
|
33
|
+
logger
|
34
|
+
end
|
35
|
+
|
36
|
+
def options
|
37
|
+
@options ||= DEFAULTS.dup
|
38
|
+
end
|
39
|
+
|
40
|
+
def options=(opts)
|
41
|
+
@options = opts
|
42
|
+
end
|
43
|
+
|
44
|
+
def configure
|
45
|
+
yield self if block_given?
|
46
|
+
end
|
47
|
+
|
48
|
+
def logger
|
49
|
+
@logger ||= default_logger
|
50
|
+
end
|
51
|
+
|
52
|
+
def logger=(log)
|
53
|
+
@logger = log
|
54
|
+
end
|
55
|
+
|
56
|
+
def engine
|
57
|
+
@engine ||= options[:engine]
|
58
|
+
end
|
59
|
+
|
60
|
+
def engine=(engine)
|
61
|
+
@engine = engine
|
62
|
+
end
|
63
|
+
|
64
|
+
def dyno_scalers
|
65
|
+
@dyno_scalers ||= {}
|
66
|
+
end
|
67
|
+
|
68
|
+
def dyno_scalers=(dyno_scalers)
|
69
|
+
@dyno_scalers = dyno_scalers
|
70
|
+
end
|
71
|
+
|
72
|
+
def add_dyno_scaler(name, options={})
|
73
|
+
dyno_scalers[name] = options
|
74
|
+
end
|
75
|
+
|
76
|
+
def min_scale_time_secs
|
77
|
+
@min_scale_time_secs ||= options[:min_scale_time_secs]
|
78
|
+
end
|
79
|
+
|
80
|
+
def min_scale_time_secs=(time_secs)
|
81
|
+
@min_scale_time_secs = time_secs
|
82
|
+
end
|
83
|
+
|
84
|
+
def heroku_app_name
|
85
|
+
@heroku_app_name ||= ENV['HEROKU_APP_NAME']
|
86
|
+
end
|
87
|
+
|
88
|
+
def heroku_app_name=(app_name)
|
89
|
+
@heroku_app_name = app_name
|
90
|
+
end
|
91
|
+
|
92
|
+
def newrelic_api_key
|
93
|
+
@newrelic_api_key ||= ENV['NEWRELIC_API_KEY']
|
94
|
+
end
|
95
|
+
|
96
|
+
def newrelic_api_key=(api_key)
|
97
|
+
@newrelic_api_key = api_key
|
98
|
+
end
|
99
|
+
|
100
|
+
def newrelic_account_id
|
101
|
+
@newrelic_account_id ||= ENV['NEWRELIC_ACCOUNT_ID']
|
102
|
+
end
|
103
|
+
|
104
|
+
def newrelic_account_id=(account_id)
|
105
|
+
@newrelic_account_id = account_id
|
106
|
+
end
|
107
|
+
|
108
|
+
def newrelic_app_id
|
109
|
+
@newrelic_app_id ||= ENV['NEWRELIC_APP_ID']
|
110
|
+
end
|
111
|
+
|
112
|
+
def newrelic_app_id=(app_id)
|
113
|
+
@newrelic_app_id = app_id
|
114
|
+
end
|
115
|
+
|
116
|
+
def sidekiq_redis_url
|
117
|
+
ENV['REDIS_URL'] || 'redis://127.0.0.1/0'
|
118
|
+
end
|
119
|
+
|
120
|
+
def sidekiq_redis_namespace
|
121
|
+
ENV['SIDEKIQ_REDIS_NAMESPACE']
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
module HerokuVector
|
127
|
+
DEFAULTS = {
|
128
|
+
:min_scale_time_secs => 5 * 60, # 5 mins
|
129
|
+
:log_level => Logger::INFO,
|
130
|
+
:engine => Engine::Heroku.new
|
131
|
+
}
|
132
|
+
|
133
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
module HerokuVector
|
2
|
+
class DynoScaler
|
3
|
+
include HerokuVector::Helper
|
4
|
+
|
5
|
+
MIN_SCALE_TIME_DELTA_SEC = 5 * 60 # 5 mins
|
6
|
+
|
7
|
+
attr_accessor :source, :last_scale_time
|
8
|
+
attr_reader :name,
|
9
|
+
:sampler, :period, :min_value, :max_value,
|
10
|
+
:min_dynos, :max_dynos,
|
11
|
+
:engine, :scale_up_by, :scale_down_by
|
12
|
+
|
13
|
+
def initialize(name, options={})
|
14
|
+
@name = name
|
15
|
+
@period = options[:period] || 60 # 1 min
|
16
|
+
sample_size = options[:sample_size] || Sampler.capacity_for_sample_period(@period)
|
17
|
+
@sampler = Sampler.new( sample_size )
|
18
|
+
@min_dynos = options[:min_dynos] || 2
|
19
|
+
@max_dynos = options[:max_dynos] || 10
|
20
|
+
@min_value = options[:min_value] || raise("DynoScaler: min_value required")
|
21
|
+
@max_value = options[:max_value] || raise("DynoScaler: max_value required")
|
22
|
+
|
23
|
+
@scale_up_by = options[:scale_up_by] || 1
|
24
|
+
@scale_down_by = options[:scale_down_by] || 1
|
25
|
+
@source = load_data_source(options)
|
26
|
+
@engine = options[:engine] || Engine::Heroku.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def load_data_source(options)
|
30
|
+
source = options[:source]
|
31
|
+
|
32
|
+
if source.is_a?(Class)
|
33
|
+
@source = source.new(options)
|
34
|
+
else
|
35
|
+
begin
|
36
|
+
clazz = HerokuVector::Source.const_get(source)
|
37
|
+
@source = clazz.new(options)
|
38
|
+
rescue
|
39
|
+
raise "DynoScaler: Invalid source class '#{source}'"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def reset
|
45
|
+
sampler.clear
|
46
|
+
@last_scale_time = nil
|
47
|
+
end
|
48
|
+
|
49
|
+
def run
|
50
|
+
begin
|
51
|
+
collect_sample
|
52
|
+
return unless enough_samples?
|
53
|
+
return if scaling_too_soon?
|
54
|
+
|
55
|
+
evaluate_and_scale
|
56
|
+
rescue => e
|
57
|
+
logger.error "#{self.name} worker.run(): #{e}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def evaluate_and_scale
|
62
|
+
num_dynos = current_size
|
63
|
+
total_min = min_value * num_dynos
|
64
|
+
total_max = max_value * num_dynos
|
65
|
+
value = current_value
|
66
|
+
|
67
|
+
logger.debug "#{self.name}: #{num_dynos} dynos - #{value} #{display_unit}"
|
68
|
+
if value < total_min
|
69
|
+
unless num_dynos <= min_dynos
|
70
|
+
logger.info "#{self.name}: #{num_dynos} dynos - #{value} #{display_unit} below #{total_min} - scaling down"
|
71
|
+
end
|
72
|
+
# Always scale down one at a time
|
73
|
+
new_amount = num_dynos - scale_down_by
|
74
|
+
scale_dynos(num_dynos, new_amount)
|
75
|
+
elsif value > total_max
|
76
|
+
unless num_dynos >= max_dynos
|
77
|
+
logger.info "#{self.name}: #{num_dynos} dynos - #{value} #{display_unit} above #{total_max} - scaling up"
|
78
|
+
end
|
79
|
+
# Scale up to N new dynos
|
80
|
+
new_amount = num_dynos + scale_up_by
|
81
|
+
scale_dynos(num_dynos, new_amount)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def current_value
|
86
|
+
sampler.mean
|
87
|
+
end
|
88
|
+
|
89
|
+
def collect_sample
|
90
|
+
sampler << source.sample
|
91
|
+
end
|
92
|
+
|
93
|
+
def current_size
|
94
|
+
engine.count_for_dyno_name(self.name)
|
95
|
+
end
|
96
|
+
|
97
|
+
def scale_dynos(current_amount, new_amount)
|
98
|
+
new_amount = normalize_dyno_increment(new_amount)
|
99
|
+
return if current_amount == new_amount
|
100
|
+
|
101
|
+
record_last_scale_event
|
102
|
+
|
103
|
+
engine.scale_dynos(self.name, new_amount)
|
104
|
+
end
|
105
|
+
|
106
|
+
def normalize_dyno_increment(amount)
|
107
|
+
[
|
108
|
+
[amount, max_dynos].min,
|
109
|
+
min_dynos
|
110
|
+
].max
|
111
|
+
end
|
112
|
+
|
113
|
+
def record_last_scale_event
|
114
|
+
@last_scale_time = Time.now
|
115
|
+
end
|
116
|
+
|
117
|
+
def enough_samples?
|
118
|
+
sampler.full?
|
119
|
+
end
|
120
|
+
|
121
|
+
def scaling_too_soon?
|
122
|
+
return false unless last_scale_time
|
123
|
+
scale_delta = Time.now - last_scale_time
|
124
|
+
|
125
|
+
MIN_SCALE_TIME_DELTA_SEC >= scale_delta
|
126
|
+
end
|
127
|
+
|
128
|
+
def display_unit
|
129
|
+
source.unit rescue 'units'
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'heroku-api'
|
2
|
+
|
3
|
+
module HerokuVector::Engine
|
4
|
+
class Heroku
|
5
|
+
include HerokuVector::Helper
|
6
|
+
|
7
|
+
CacheEntry = Struct.new(:data, :expires_at)
|
8
|
+
|
9
|
+
attr_accessor :app, :heroku
|
10
|
+
|
11
|
+
def initialize(options={})
|
12
|
+
options = { app: HerokuVector.heroku_app_name }.merge(options)
|
13
|
+
|
14
|
+
@app = options[:app]
|
15
|
+
@heroku = ::Heroku::API.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_dynos_by_name_cached
|
19
|
+
if @dynos_by_name && @dynos_by_name.expires_at > Time.now
|
20
|
+
return @dynos_by_name.data
|
21
|
+
end
|
22
|
+
|
23
|
+
@dynos_by_name = begin
|
24
|
+
expires_at = Time.now + 60 # 1 min from now
|
25
|
+
CacheEntry.new(get_dynos_by_name, expires_at)
|
26
|
+
rescue
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
@dynos_by_name.data
|
30
|
+
end
|
31
|
+
|
32
|
+
def get_dynos_by_name
|
33
|
+
dyno_types = get_dyno_types
|
34
|
+
dyno_types.index_by {|dyno| dyno["name"] }
|
35
|
+
end
|
36
|
+
|
37
|
+
def get_dyno_types
|
38
|
+
response = heroku.get_dyno_types(app)
|
39
|
+
assert_heroku_api_success(response)
|
40
|
+
response.body
|
41
|
+
end
|
42
|
+
|
43
|
+
def count_for_dyno_name(dyno_name)
|
44
|
+
dyno = get_dynos_by_name_cached[dyno_name]
|
45
|
+
return 0 unless dyno
|
46
|
+
|
47
|
+
dyno['quantity'] || 0
|
48
|
+
end
|
49
|
+
|
50
|
+
def scale_dynos(dyno_name, count)
|
51
|
+
response = heroku.post_ps_scale(app, dyno_name, count)
|
52
|
+
assert_heroku_api_success(response)
|
53
|
+
logger.info "Heroku.scale_dynos(#{dyno_name}, #{count})"
|
54
|
+
|
55
|
+
response.body
|
56
|
+
end
|
57
|
+
|
58
|
+
def assert_heroku_api_success(response)
|
59
|
+
raise 'Invalid Heroku API Response' unless response
|
60
|
+
raise "Error #{response.status} from Heroku API" unless response.status == 200
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
|
2
|
+
module HerokuVector
|
3
|
+
module Helper
|
4
|
+
def self.included(target)
|
5
|
+
target.send(:include, InstanceMethods)
|
6
|
+
target.extend ClassMethods
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
end
|
11
|
+
|
12
|
+
module InstanceMethods
|
13
|
+
def logger
|
14
|
+
HerokuVector.logger
|
15
|
+
end
|
16
|
+
|
17
|
+
def round_to_one_decimal(value)
|
18
|
+
((value * 10.0).floor.to_f / 10.0)
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
|
2
|
+
module HerokuVector
|
3
|
+
class ProcessManager
|
4
|
+
include HerokuVector::Helper
|
5
|
+
|
6
|
+
attr_accessor :options
|
7
|
+
|
8
|
+
def initialize(options = {})
|
9
|
+
@options = options
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
daemonize if options[:daemonize]
|
14
|
+
redirect_to_logfile if options[:logfile]
|
15
|
+
write_pid
|
16
|
+
|
17
|
+
worker = Worker.new(options)
|
18
|
+
worker.run
|
19
|
+
end
|
20
|
+
|
21
|
+
def daemonize
|
22
|
+
return unless options[:daemonize]
|
23
|
+
|
24
|
+
raise ArgumentError, "Daemonized mode requires a logfile" unless options[:logfile]
|
25
|
+
files_to_reopen = []
|
26
|
+
ObjectSpace.each_object(File) do |file|
|
27
|
+
files_to_reopen << file unless file.closed?
|
28
|
+
end
|
29
|
+
|
30
|
+
::Process.daemon(true, true)
|
31
|
+
|
32
|
+
files_to_reopen.each do |file|
|
33
|
+
begin
|
34
|
+
file.reopen file.path, "a+"
|
35
|
+
file.sync = true
|
36
|
+
rescue ::Exception
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def redirect_to_logfile
|
42
|
+
[$stdout, $stderr].each do |io|
|
43
|
+
File.open(options[:logfile], 'ab') do |f|
|
44
|
+
io.reopen(f)
|
45
|
+
end
|
46
|
+
io.sync = true
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def write_pid
|
51
|
+
return unless path = options[:pidfile]
|
52
|
+
|
53
|
+
pidfile = File.expand_path(path)
|
54
|
+
logger.info "Writing pidfile #{pidfile}"
|
55
|
+
File.open(pidfile, 'w') do |f|
|
56
|
+
f.puts ::Process.pid
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|