our-eel-hacks 0.0.10 → 0.0.11
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/our-eel-hacks/autoscaler.rb +50 -17
- data/spec/autoscaler.rb +20 -13
- data/spec/rack.rb +0 -1
- metadata +8 -8
@@ -68,6 +68,8 @@ module OurEelHacks
|
|
68
68
|
@dynos = nil
|
69
69
|
@soft_side = nil
|
70
70
|
|
71
|
+
@memoed_dyno_info = nil
|
72
|
+
|
71
73
|
@last_scaled = Time.at(0)
|
72
74
|
@entered_soft = Time.at(0)
|
73
75
|
@last_reading = nil
|
@@ -80,8 +82,14 @@ module OurEelHacks
|
|
80
82
|
@max_dynos = 10
|
81
83
|
@lower_limits = LowerLimit.new(5, 1)
|
82
84
|
@upper_limits = UpperLimit.new(30, 50)
|
83
|
-
|
84
|
-
@
|
85
|
+
|
86
|
+
@soft_duration = 10000
|
87
|
+
@scaling_frequency = 5000
|
88
|
+
@heroku_rate_limit = 80_000
|
89
|
+
@heroku_rate_limit_margin = 0.1
|
90
|
+
|
91
|
+
@millis_til_next_scale = nil
|
92
|
+
|
85
93
|
@logger = NullLogger.new
|
86
94
|
end
|
87
95
|
|
@@ -90,15 +98,20 @@ module OurEelHacks
|
|
90
98
|
check_settings
|
91
99
|
logger.info{ "Autoscaler configured for #{flavor || "{{unknown flavor}}"}"}
|
92
100
|
|
93
|
-
update_dynos(Time.now)
|
94
|
-
logger.debug{ self.inspect }
|
101
|
+
update_dynos(dyno_info.count, Time.now)
|
95
102
|
end
|
96
103
|
|
104
|
+
MILLIS_PER_DAY = 24 * 60 * 60 * 1000
|
97
105
|
def check_settings
|
98
106
|
errors = []
|
99
107
|
errors << "No heroku api key set" if @heroku_api_key.nil?
|
100
108
|
errors << "No app name set" if @app_name.nil?
|
101
109
|
errors << "No process type set" if @ps_type.nil?
|
110
|
+
if (MILLIS_PER_DAY / @heroku_rate_limit) *
|
111
|
+
(1.0 - @heroku_rate_limit_margin) *
|
112
|
+
API_CALLS_PER_SCALE > @scaling_frequency
|
113
|
+
errors << "Scaling frequency will lock up Heroku"
|
114
|
+
end
|
102
115
|
unless errors.empty?
|
103
116
|
logger.warn{ "Problems configuring Autoscaler: #{errors.inspect}" }
|
104
117
|
raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ")
|
@@ -107,7 +120,7 @@ module OurEelHacks
|
|
107
120
|
|
108
121
|
attr_accessor :min_dynos, :max_dynos, :lower_limits, :upper_limits, :ps_type,
|
109
122
|
:soft_duration, :scaling_frequency, :logger, :heroku_api_key, :app_name
|
110
|
-
attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side
|
123
|
+
attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side, :millis_til_next_scale
|
111
124
|
|
112
125
|
def elapsed(start, finish)
|
113
126
|
seconds = finish.to_i - start.to_i
|
@@ -117,22 +130,29 @@ module OurEelHacks
|
|
117
130
|
return diff
|
118
131
|
end
|
119
132
|
|
133
|
+
API_CALLS_PER_SCALE = 2
|
120
134
|
def scale(metric)
|
121
135
|
logger.debug{ "Scaling request for #{@ps_type}: metric is: #{metric}" }
|
122
136
|
moment = Time.now
|
123
|
-
if elapsed(last_scaled, moment) <
|
124
|
-
logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than
|
137
|
+
if elapsed(last_scaled, moment) < millis_til_next_scale
|
138
|
+
logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than computed #{millis_til_next_scale}" }
|
125
139
|
return
|
126
140
|
end
|
127
141
|
|
142
|
+
clear_dyno_info
|
143
|
+
|
144
|
+
starting_wait = millis_til_next_scale
|
145
|
+
|
146
|
+
update_dynos(dyno_info.count, moment)
|
147
|
+
|
128
148
|
target_dynos = target_scale(metric, moment)
|
129
149
|
|
130
150
|
target_dynos = [[target_dynos, max_dynos].min, min_dynos].max
|
131
151
|
logger.debug{ "Target dynos at: #{min_dynos}/#{target_dynos}/#{max_dynos} (vs. current: #{@dynos})" }
|
132
152
|
|
133
|
-
set_dynos(target_dynos)
|
153
|
+
set_dynos(target_dynos, moment)
|
134
154
|
|
135
|
-
|
155
|
+
break_cadence(starting_wait)
|
136
156
|
rescue => ex
|
137
157
|
logger.warn{ "Problem scaling: #{ex.inspect}" }
|
138
158
|
end
|
@@ -173,16 +193,15 @@ module OurEelHacks
|
|
173
193
|
return 0
|
174
194
|
end
|
175
195
|
|
176
|
-
def
|
177
|
-
|
178
|
-
|
179
|
-
dyno["process"] =~ regexp
|
196
|
+
def break_cadence(starting_wait)
|
197
|
+
if starting_wait > millis_til_next_scale
|
198
|
+
@millis_til_next_scale = rand((starting_wait..@millis_til_next_scale))
|
180
199
|
end
|
181
200
|
end
|
182
201
|
|
183
|
-
def update_dynos(moment)
|
184
|
-
new_value = dyno_info.count
|
202
|
+
def update_dynos(new_value, moment)
|
185
203
|
if new_value != dynos
|
204
|
+
@millis_til_next_scale = scaling_frequency * new_value
|
186
205
|
@last_scaled = moment
|
187
206
|
@entered_soft = moment
|
188
207
|
end
|
@@ -190,6 +209,20 @@ module OurEelHacks
|
|
190
209
|
@last_reading = moment
|
191
210
|
end
|
192
211
|
|
212
|
+
def clear_dyno_info
|
213
|
+
@memoed_dyno_info = nil
|
214
|
+
end
|
215
|
+
|
216
|
+
def dyno_info
|
217
|
+
return @memoed_dyno_info ||=
|
218
|
+
begin
|
219
|
+
regexp = /^#{ps_type}[.].*/
|
220
|
+
heroku.ps(app_name).find_all do |dyno|
|
221
|
+
dyno["process"] =~ regexp
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
193
226
|
def dynos_stable?
|
194
227
|
return dyno_info.all? do |dyno|
|
195
228
|
dyno["state"] == "up"
|
@@ -204,7 +237,7 @@ module OurEelHacks
|
|
204
237
|
end
|
205
238
|
end
|
206
239
|
|
207
|
-
def set_dynos(count)
|
240
|
+
def set_dynos(count,moment)
|
208
241
|
if count == dynos
|
209
242
|
logger.debug{ "Not scaling: #{count} ?= #{dynos}" }
|
210
243
|
return
|
@@ -216,7 +249,7 @@ module OurEelHacks
|
|
216
249
|
end
|
217
250
|
logger.info{ "Scaling from #{dynos} to #{count} dynos for #{ps_type}" }
|
218
251
|
heroku.ps_scale(app_name, :type => ps_type, :qty => count)
|
219
|
-
|
252
|
+
update_dynos(count, moment)
|
220
253
|
end
|
221
254
|
end
|
222
255
|
end
|
data/spec/autoscaler.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'spec_helper'
|
2
1
|
require 'our-eel-hacks/rack'
|
3
2
|
require 'logger'
|
4
3
|
|
@@ -18,11 +17,11 @@ describe OurEelHacks::Autoscaler do
|
|
18
17
|
end
|
19
18
|
|
20
19
|
let :scaling_freq do
|
21
|
-
|
20
|
+
2500
|
22
21
|
end
|
23
22
|
|
24
23
|
let :soft_dur do
|
25
|
-
|
24
|
+
5000
|
26
25
|
end
|
27
26
|
|
28
27
|
let :ideal_value do
|
@@ -45,6 +44,14 @@ describe OurEelHacks::Autoscaler do
|
|
45
44
|
-10
|
46
45
|
end
|
47
46
|
|
47
|
+
let :dyno_count do
|
48
|
+
3
|
49
|
+
end
|
50
|
+
|
51
|
+
let :expected_scale_frequency do
|
52
|
+
scaling_freq * dyno_count
|
53
|
+
end
|
54
|
+
|
48
55
|
let! :starting_time do
|
49
56
|
Time.now
|
50
57
|
end
|
@@ -83,7 +90,7 @@ describe OurEelHacks::Autoscaler do
|
|
83
90
|
end
|
84
91
|
|
85
92
|
it "should get a count of dynos at start" do
|
86
|
-
autoscaler.dynos.should ==
|
93
|
+
autoscaler.dynos.should == dyno_count #comes from the VCR cassette
|
87
94
|
end
|
88
95
|
|
89
96
|
before :each do
|
@@ -95,14 +102,14 @@ describe OurEelHacks::Autoscaler do
|
|
95
102
|
describe "scaling frequency" do
|
96
103
|
|
97
104
|
it "should not scale too soon" do
|
98
|
-
time_adjust(
|
105
|
+
time_adjust(expected_scale_frequency - 5)
|
99
106
|
|
100
107
|
heroku.should_not_receive(:ps_scale)
|
101
108
|
autoscaler.scale(hard_high)
|
102
109
|
end
|
103
110
|
|
104
111
|
it "should scale up if time has elapsed and hard limit exceeded" do
|
105
|
-
time_adjust(
|
112
|
+
time_adjust(expected_scale_frequency + 5)
|
106
113
|
|
107
114
|
heroku.should_receive(:ps_scale).with(app_name, hash_including(:qty => 4))
|
108
115
|
autoscaler.scale(hard_high)
|
@@ -111,7 +118,7 @@ describe OurEelHacks::Autoscaler do
|
|
111
118
|
|
112
119
|
describe "hard limits" do
|
113
120
|
before :each do
|
114
|
-
time_adjust(
|
121
|
+
time_adjust(expected_scale_frequency + 5)
|
115
122
|
end
|
116
123
|
|
117
124
|
it "should scale down if hard lower limit exceeded" do
|
@@ -122,13 +129,13 @@ describe OurEelHacks::Autoscaler do
|
|
122
129
|
|
123
130
|
describe "soft upper limit" do
|
124
131
|
before :each do
|
125
|
-
time_adjust(
|
132
|
+
time_adjust(expected_scale_frequency * 2)
|
126
133
|
autoscaler.scale(soft_high)
|
127
134
|
end
|
128
135
|
|
129
136
|
describe "if soft_duration hasn't elapsed" do
|
130
137
|
before :each do
|
131
|
-
time_adjust((
|
138
|
+
time_adjust((expected_scale_frequency * 2) + soft_dur - 5)
|
132
139
|
heroku.should_not_receive(:ps_scale)
|
133
140
|
end
|
134
141
|
|
@@ -143,7 +150,7 @@ describe OurEelHacks::Autoscaler do
|
|
143
150
|
|
144
151
|
describe "if soft_duration has elapsed" do
|
145
152
|
before :each do
|
146
|
-
time_adjust(
|
153
|
+
time_adjust(expected_scale_frequency * 2 + soft_dur + 5)
|
147
154
|
end
|
148
155
|
|
149
156
|
it "should scale up if above upper soft limit" do
|
@@ -160,13 +167,13 @@ describe OurEelHacks::Autoscaler do
|
|
160
167
|
|
161
168
|
describe "soft lower limit" do
|
162
169
|
before :each do
|
163
|
-
time_adjust(
|
170
|
+
time_adjust(expected_scale_frequency * 2)
|
164
171
|
autoscaler.scale(soft_low)
|
165
172
|
end
|
166
173
|
|
167
174
|
describe "if soft_duration hasn't elapsed" do
|
168
175
|
before :each do
|
169
|
-
time_adjust(
|
176
|
+
time_adjust(expected_scale_frequency * 2 + soft_dur - 5)
|
170
177
|
heroku.should_not_receive(:ps_scale)
|
171
178
|
end
|
172
179
|
|
@@ -181,7 +188,7 @@ describe OurEelHacks::Autoscaler do
|
|
181
188
|
|
182
189
|
describe "if soft_duration has elapsed" do
|
183
190
|
before :each do
|
184
|
-
time_adjust(
|
191
|
+
time_adjust(expected_scale_frequency * 2 + soft_dur + 5)
|
185
192
|
end
|
186
193
|
|
187
194
|
it "should not scale up even if above upper soft limit" do
|
data/spec/rack.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: our-eel-hacks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-05-
|
12
|
+
date: 2012-05-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: corundum
|
16
|
-
requirement: &
|
16
|
+
requirement: &75630260 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 0.0.1
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *75630260
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: heroku
|
27
|
-
requirement: &
|
27
|
+
requirement: &75643080 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>'
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
- 0
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *75643080
|
38
38
|
description: ! " Middleware for Rack and Sidekiq to scale heroku.\n\n A heroku process
|
39
39
|
knows everything it needs in order to scale itself. A little configuration, and
|
40
40
|
you're set.\n"
|
@@ -66,7 +66,7 @@ rdoc_options:
|
|
66
66
|
- --main
|
67
67
|
- doc/README
|
68
68
|
- --title
|
69
|
-
- our-eel-hacks-0.0.
|
69
|
+
- our-eel-hacks-0.0.11 RDoc
|
70
70
|
require_paths:
|
71
71
|
- lib/
|
72
72
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -77,7 +77,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
77
77
|
version: '0'
|
78
78
|
segments:
|
79
79
|
- 0
|
80
|
-
hash:
|
80
|
+
hash: 838375187
|
81
81
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
82
|
none: false
|
83
83
|
requirements:
|