dynamo-autoscale 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/Gemfile +1 -0
- data/Gemfile.lock +13 -8
- data/README.md +5 -1
- data/bin/dynamo-autoscale +30 -19
- data/config/environment/common.rb +73 -18
- data/config/environment/test.rb +25 -2
- data/config/services/aws.rb +18 -2
- data/config/services/logger.rb +20 -21
- data/lib/dynamo-autoscale/actioner.rb +1 -0
- data/lib/dynamo-autoscale/dynamo_actioner.rb +3 -2
- data/lib/dynamo-autoscale/fake_poller.rb +40 -0
- data/lib/dynamo-autoscale/log_collector.rb +18 -0
- data/lib/dynamo-autoscale/logger.rb +5 -1
- data/lib/dynamo-autoscale/metrics.rb +11 -28
- data/lib/dynamo-autoscale/poller.rb +17 -6
- data/lib/dynamo-autoscale/random_data_generator.rb +51 -0
- data/lib/dynamo-autoscale/scale_report.rb +2 -2
- data/lib/dynamo-autoscale/table_tracker.rb +56 -13
- data/lib/dynamo-autoscale/unit_cost.rb +42 -14
- data/lib/dynamo-autoscale/version.rb +1 -1
- data/script/historic_data +1 -1
- data/script/random_test +68 -0
- data/script/test +4 -3
- data/spec/config_spec.rb +72 -0
- data/spec/dispatcher_spec.rb +56 -0
- data/spec/helpers/environment_helper.rb +15 -0
- data/spec/helpers/logger.rb +20 -0
- data/spec/poller_spec.rb +39 -0
- data/spec/spec_helper.rb +0 -3
- data/spec/table_tracker_spec.rb +163 -0
- data/spec/unit_cost_spec.rb +39 -0
- metadata +13 -3
@@ -0,0 +1,40 @@
|
|
1
|
+
module DynamoAutoscale
|
2
|
+
# This class allows you to specify exactly what data to send through the
|
3
|
+
# poller.
|
4
|
+
#
|
5
|
+
# Example:
|
6
|
+
#
|
7
|
+
# DynamoAutoscale.poller_class = FakePoller
|
8
|
+
# DynamoAutoscale.poller_opts = {
|
9
|
+
# data: {
|
10
|
+
# Time.now => {
|
11
|
+
# provisioned_reads: 100,
|
12
|
+
# provisioned_writes: 100,
|
13
|
+
# consumed_reads: 55.6,
|
14
|
+
# consumed_writes: 12.7,
|
15
|
+
# },
|
16
|
+
# Time.now + 15.minutes => {
|
17
|
+
# provisioned_reads: 100,
|
18
|
+
# provisioned_writes: 100,
|
19
|
+
# consumed_reads: 45.9,
|
20
|
+
# consumed_writes: 7.1,
|
21
|
+
# },
|
22
|
+
# }
|
23
|
+
# }
|
24
|
+
class FakePoller < Poller
|
25
|
+
def initialize *args
|
26
|
+
super(*args)
|
27
|
+
opts = args.last.is_a?(Hash) ? args.last : {}
|
28
|
+
|
29
|
+
@data = opts[:data]
|
30
|
+
end
|
31
|
+
|
32
|
+
def poll tables, &block
|
33
|
+
@data.each do |key, value|
|
34
|
+
tables.each do |table_name|
|
35
|
+
block.call(table_name, key => value)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module DynamoAutoscale
|
2
|
+
class LogCollector < ::Logger
|
3
|
+
attr_reader :messages
|
4
|
+
|
5
|
+
def initialize log_to
|
6
|
+
super
|
7
|
+
|
8
|
+
@messages = Hash.new { |h, k| h[k] = [] }
|
9
|
+
end
|
10
|
+
|
11
|
+
[:debug, :info, :warn, :error, :fatal].each do |level|
|
12
|
+
define_method level do |message|
|
13
|
+
@messages[level] << message
|
14
|
+
super message
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -5,13 +5,17 @@ module DynamoAutoscale
|
|
5
5
|
end
|
6
6
|
|
7
7
|
def self.logger
|
8
|
-
@@logger
|
8
|
+
@@logger ||= ::Logger.new(STDOUT)
|
9
9
|
end
|
10
10
|
|
11
11
|
def logger
|
12
12
|
DynamoAutoscale::Logger.logger
|
13
13
|
end
|
14
14
|
|
15
|
+
def logger= new_logger
|
16
|
+
DynamoAutoscale::Logger.logger = new_logger
|
17
|
+
end
|
18
|
+
|
15
19
|
def self.included base
|
16
20
|
base.extend DynamoAutoscale::Logger
|
17
21
|
end
|
@@ -3,32 +3,10 @@ module DynamoAutoscale
|
|
3
3
|
include DynamoAutoscale::Logger
|
4
4
|
|
5
5
|
DEFAULT_OPTS = {
|
6
|
-
namespace:
|
7
|
-
period:
|
8
|
-
# metric_name: metric,
|
9
|
-
# start_time: (NOW - 3600).iso8601,
|
10
|
-
# end_time: NOW.iso8601,
|
11
|
-
# dimensions: [{
|
12
|
-
# name: "TableName", value: TABLE_NAME,
|
13
|
-
# }],
|
6
|
+
namespace: 'AWS/DynamoDB',
|
7
|
+
period: 300,
|
14
8
|
}
|
15
9
|
|
16
|
-
# Returns a CloudWatch client object for a given region. If no region
|
17
|
-
# exists, the region defaults to whatever is in
|
18
|
-
# DynamoAutoscale::DEFAULT_AWS_REGION.
|
19
|
-
#
|
20
|
-
# CloudWatch client documentation:
|
21
|
-
# https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/cloud_watch/client.rb
|
22
|
-
def self.client region = nil
|
23
|
-
@client ||= Hash.new do |hash, _region|
|
24
|
-
hash[_region] = AWS::CloudWatch.new({
|
25
|
-
cloud_watch_endpoint: "monitoring.#{_region}.amazonaws.com",
|
26
|
-
}).client
|
27
|
-
end
|
28
|
-
|
29
|
-
@client[region || DEFAULT_AWS_REGION]
|
30
|
-
end
|
31
|
-
|
32
10
|
# Returns a hash of timeseries data. Looks a bit like this:
|
33
11
|
#
|
34
12
|
# {
|
@@ -171,22 +149,27 @@ module DynamoAutoscale
|
|
171
149
|
# A base method that gets called by wrapper methods defined above. Makes a
|
172
150
|
# call to CloudWatch, getting statistics on whatever metric is given.
|
173
151
|
def self.metric_statistics table_name, opts = {}
|
174
|
-
region = opts.delete :region
|
175
152
|
opts = DEFAULT_OPTS.merge({
|
176
153
|
dimensions: [{ name: "TableName", value: table_name }],
|
177
154
|
start_time: 1.hour.ago,
|
178
155
|
end_time: Time.now,
|
179
156
|
}).merge(opts)
|
180
157
|
|
181
|
-
if opts[:start_time] and opts[:start_time].respond_to?
|
158
|
+
if opts[:start_time] and opts[:start_time].respond_to?(:iso8601)
|
182
159
|
opts[:start_time] = opts[:start_time].iso8601
|
183
160
|
end
|
184
161
|
|
185
|
-
if opts[:end_time] and opts[:end_time].respond_to?
|
162
|
+
if opts[:end_time] and opts[:end_time].respond_to?(:iso8601)
|
186
163
|
opts[:end_time] = opts[:end_time].iso8601
|
187
164
|
end
|
188
165
|
|
189
|
-
client
|
166
|
+
client.get_metric_statistics(opts)[:datapoints]
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def self.client
|
172
|
+
@@client ||= AWS::CloudWatch.new.client
|
190
173
|
end
|
191
174
|
end
|
192
175
|
end
|
@@ -30,12 +30,23 @@ module DynamoAutoscale
|
|
30
30
|
end.sort!.uniq
|
31
31
|
|
32
32
|
times.each do |time|
|
33
|
-
datum = {
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
33
|
+
datum = {}
|
34
|
+
|
35
|
+
if data[:provisioned_writes] and data[:provisioned_writes][time]
|
36
|
+
datum[:provisioned_writes] = data[:provisioned_writes][time]
|
37
|
+
end
|
38
|
+
|
39
|
+
if data[:provisioned_reads] and data[:provisioned_reads][time]
|
40
|
+
datum[:provisioned_reads] = data[:provisioned_reads][time]
|
41
|
+
end
|
42
|
+
|
43
|
+
if data[:consumed_writes] and data[:consumed_writes][time]
|
44
|
+
datum[:consumed_writes] = data[:consumed_writes][time]
|
45
|
+
end
|
46
|
+
|
47
|
+
if data[:consumed_reads] and data[:consumed_reads][time]
|
48
|
+
datum[:consumed_reads] = data[:consumed_reads][time]
|
49
|
+
end
|
39
50
|
|
40
51
|
filters.each { |filter| filter.call(table, time, datum) }
|
41
52
|
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module DynamoAutoscale
|
2
|
+
# You can use this class as the DynamoAutoscale.poller global and it will just
|
3
|
+
# generate a given number of random data points at 15 minutes intervals. Nice
|
4
|
+
# for testing purposes.
|
5
|
+
class RandomDataGenerator < Poller
|
6
|
+
def initialize *args
|
7
|
+
super(*args)
|
8
|
+
opts = args.last.is_a?(Hash) ? args.last : {}
|
9
|
+
|
10
|
+
@num_points = opts[:num_points] || 100
|
11
|
+
@start_time = opts[:start_time] || Time.now
|
12
|
+
@current_time = @start_time
|
13
|
+
@provisioned_reads = opts[:provisioned_reads] || 600
|
14
|
+
@provisioned_writes = opts[:provisioned_writes] || 600
|
15
|
+
|
16
|
+
srand(@start_time.to_i)
|
17
|
+
end
|
18
|
+
|
19
|
+
def poll tables, &block
|
20
|
+
# Give each table its initial provisioned reads and writes before starting
|
21
|
+
# the random data generation.
|
22
|
+
tables.each do |table_name|
|
23
|
+
block.call(table_name, {
|
24
|
+
provisioned_reads: {
|
25
|
+
@start_time => @provisioned_reads,
|
26
|
+
},
|
27
|
+
provisioned_writes: {
|
28
|
+
@start_time => @provisioned_writes,
|
29
|
+
},
|
30
|
+
})
|
31
|
+
end
|
32
|
+
|
33
|
+
@num_points.times do
|
34
|
+
tables.each do |table_name|
|
35
|
+
# Give each table varying figures for consumed reads and writes that
|
36
|
+
# hover between 0 and their provisioned values multiplied by 2.
|
37
|
+
block.call(table_name, {
|
38
|
+
consumed_reads: {
|
39
|
+
@current_time => rand * @provisioned_reads * 2,
|
40
|
+
},
|
41
|
+
consumed_writes: {
|
42
|
+
@current_time => rand * @provisioned_writes * 2,
|
43
|
+
},
|
44
|
+
})
|
45
|
+
|
46
|
+
@current_time += 15.minutes
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -2,7 +2,7 @@ module DynamoAutoscale
|
|
2
2
|
class ScaleReport
|
3
3
|
include DynamoAutoscale::Logger
|
4
4
|
|
5
|
-
TEMPLATE =
|
5
|
+
TEMPLATE = DynamoAutoscale.template_dir('scale_report_email.erb')
|
6
6
|
|
7
7
|
def initialize table
|
8
8
|
@table = table
|
@@ -48,7 +48,7 @@ module DynamoAutoscale
|
|
48
48
|
|
49
49
|
def formatted_scale_event(scale_event)
|
50
50
|
max_length = max_metric_length(scale_event)
|
51
|
-
|
51
|
+
|
52
52
|
['reads', 'writes'].map do |type|
|
53
53
|
next unless scale_event.has_key? "#{type}_from".to_sym
|
54
54
|
|
@@ -198,6 +198,30 @@ module DynamoAutoscale
|
|
198
198
|
end
|
199
199
|
end
|
200
200
|
|
201
|
+
def wasted_write_cost
|
202
|
+
UnitCost.write(wasted_write_units)
|
203
|
+
end
|
204
|
+
|
205
|
+
def wasted_read_cost
|
206
|
+
UnitCost.read(wasted_read_units)
|
207
|
+
end
|
208
|
+
|
209
|
+
def lost_write_cost
|
210
|
+
UnitCost.write(lost_write_units)
|
211
|
+
end
|
212
|
+
|
213
|
+
def lost_read_cost
|
214
|
+
UnitCost.read(lost_read_units)
|
215
|
+
end
|
216
|
+
|
217
|
+
def total_write_cost
|
218
|
+
UnitCost.write(total_write_units)
|
219
|
+
end
|
220
|
+
|
221
|
+
def total_read_cost
|
222
|
+
UnitCost.read(total_read_units)
|
223
|
+
end
|
224
|
+
|
201
225
|
def wasted_read_percent
|
202
226
|
(wasted_read_units / total_read_units) * 100.0
|
203
227
|
end
|
@@ -243,7 +267,7 @@ module DynamoAutoscale
|
|
243
267
|
# end
|
244
268
|
|
245
269
|
def to_csv! opts = {}
|
246
|
-
path = opts[:path] or
|
270
|
+
path = opts[:path] or DynamoAutoscale.root_dir("#{self.name}.csv")
|
247
271
|
|
248
272
|
CSV.open(path, 'w') do |csv|
|
249
273
|
csv << [
|
@@ -271,7 +295,7 @@ module DynamoAutoscale
|
|
271
295
|
def graph! opts = {}
|
272
296
|
data_tmp = File.join(Dir.tmpdir, 'data.csv')
|
273
297
|
png_tmp = opts[:path] || File.join(Dir.tmpdir, 'graph.png')
|
274
|
-
r_script =
|
298
|
+
r_script = DynamoAutoscale.rlib_dir('dynamodb_graph.r')
|
275
299
|
|
276
300
|
to_csv!(path: data_tmp)
|
277
301
|
|
@@ -296,7 +320,7 @@ module DynamoAutoscale
|
|
296
320
|
def scatterplot_for! metric
|
297
321
|
data_tmp = File.join(Dir.tmpdir, 'data.csv')
|
298
322
|
png_tmp = File.join(Dir.tmpdir, 'boxplot.png')
|
299
|
-
r_script =
|
323
|
+
r_script = DynamoAutoscale.rlib_dir('dynamodb_boxplot.r')
|
300
324
|
|
301
325
|
to_csv!(data_tmp)
|
302
326
|
|
@@ -309,16 +333,35 @@ module DynamoAutoscale
|
|
309
333
|
end
|
310
334
|
end
|
311
335
|
|
312
|
-
def report!
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
336
|
+
def report! opts = {}
|
337
|
+
opts[:metric] ||= :units
|
338
|
+
|
339
|
+
unless [:cost, :units].include?(opts[:metric])
|
340
|
+
raise ArgumentError.new("The :metric option must be one of :cost or " +
|
341
|
+
":units")
|
342
|
+
end
|
343
|
+
|
344
|
+
if opts[:metric] == :units
|
345
|
+
puts " Table: #{name}"
|
346
|
+
puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
|
347
|
+
puts " Total r/units: #{total_read_units.round(2)}"
|
348
|
+
puts " Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
|
349
|
+
puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
|
350
|
+
puts " Total w/units: #{total_write_units.round(2)}"
|
351
|
+
puts " Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
|
352
|
+
puts " Upscales: #{DynamoAutoscale.actioners[self].upscales}"
|
353
|
+
puts " Downscales: #{DynamoAutoscale.actioners[self].downscales}"
|
354
|
+
elsif opts[:metric] == :cost
|
355
|
+
puts " Table: #{name}"
|
356
|
+
puts "Wasted r/cost: $#{wasted_read_cost.round(2)} (#{wasted_read_percent.round(2)}%)"
|
357
|
+
puts " Total r/cost: $#{total_read_cost.round(2)}"
|
358
|
+
puts " Lost r/cost: $#{lost_read_cost.round(2)} (#{lost_read_percent.round(2)}%)"
|
359
|
+
puts "Wasted w/cost: $#{wasted_write_cost.round(2)} (#{wasted_write_percent.round(2)}%)"
|
360
|
+
puts " Total w/cost: $#{total_write_cost.round(2)}"
|
361
|
+
puts " Lost w/cost: $#{lost_write_cost.round(2)} (#{lost_write_percent.round(2)}%)"
|
362
|
+
puts " Upscales: #{DynamoAutoscale.actioners[self].upscales}"
|
363
|
+
puts " Downscales: #{DynamoAutoscale.actioners[self].downscales}"
|
364
|
+
end
|
322
365
|
end
|
323
366
|
|
324
367
|
private
|
@@ -2,40 +2,68 @@ module DynamoAutoscale
|
|
2
2
|
class UnitCost
|
3
3
|
# Pricing information obtained from: http://aws.amazon.com/dynamodb/pricing/
|
4
4
|
HOURLY_PRICING = {
|
5
|
-
'us-east-1' => {
|
5
|
+
'us-east-1' => { # US East (Northern Virginia)
|
6
6
|
read: { dollars: 0.0065, per: 50 },
|
7
7
|
write: { dollars: 0.0065, per: 10 },
|
8
8
|
},
|
9
|
-
'us-west-1' => {
|
9
|
+
'us-west-1' => { # US West (Northern California)
|
10
10
|
read: { dollars: 0.0065, per: 50 },
|
11
11
|
write: { dollars: 0.0065, per: 10 },
|
12
12
|
},
|
13
|
+
'us-west-2' => { # US West (Oregon)
|
14
|
+
read: { dollars: 0.0065, per: 50 },
|
15
|
+
write: { dollars: 0.0065, per: 10 },
|
16
|
+
},
|
17
|
+
'ap-southeast-1' => { # Asia Pacific (Singapore)
|
18
|
+
read: { dollars: 0.0074, per: 50 },
|
19
|
+
write: { dollars: 0.0074, per: 10 },
|
20
|
+
},
|
21
|
+
'ap-southeast-2' => { # Asia Pacific (Sydney)
|
22
|
+
read: { dollars: 0.0074, per: 50 },
|
23
|
+
write: { dollars: 0.0074, per: 10 },
|
24
|
+
},
|
25
|
+
'ap-northeast-1' => { # Asia Pacific (Tokyo)
|
26
|
+
read: { dollars: 0.0078, per: 50 },
|
27
|
+
write: { dollars: 0.0078, per: 10 },
|
28
|
+
},
|
29
|
+
'eu-west-1' => { # EU (Ireland)
|
30
|
+
read: { dollars: 0.00735, per: 50 },
|
31
|
+
write: { dollars: 0.00735, per: 10 },
|
32
|
+
},
|
33
|
+
'sa-east-1' => { # South America (Sao Paulo)
|
34
|
+
read: { dollars: 0.00975, per: 50 },
|
35
|
+
write: { dollars: 0.00975, per: 10 },
|
36
|
+
},
|
13
37
|
}
|
14
38
|
|
15
|
-
# Returns the cost of N read units for an hour in
|
16
|
-
#
|
17
|
-
# DynamoAutoscale::DEFAULT_AWS_REGION.
|
39
|
+
# Returns the cost of N read units for an hour in the region given by
|
40
|
+
# AWS.config.region
|
18
41
|
#
|
19
42
|
# Example:
|
20
43
|
#
|
21
|
-
# DynamoAutoscale::UnitCost.read(500
|
44
|
+
# DynamoAutoscale::UnitCost.read(500)
|
22
45
|
# #=> 0.065
|
23
46
|
def self.read units, opts = {}
|
24
|
-
pricing = HOURLY_PRICING[
|
25
|
-
|
47
|
+
if pricing = HOURLY_PRICING[AWS.config.region]
|
48
|
+
((units / pricing[:read][:per].to_f) * pricing[:read][:dollars])
|
49
|
+
else
|
50
|
+
nil
|
51
|
+
end
|
26
52
|
end
|
27
53
|
|
28
|
-
# Returns the cost of N write units for an hour in
|
29
|
-
#
|
30
|
-
# DynamoAutoscale::DEFAULT_AWS_REGION.
|
54
|
+
# Returns the cost of N write units for an hour in the region given by
|
55
|
+
# AWS.config.region.
|
31
56
|
#
|
32
57
|
# Example:
|
33
58
|
#
|
34
|
-
# DynamoAutoscale::UnitCost.write(500
|
59
|
+
# DynamoAutoscale::UnitCost.write(500)
|
35
60
|
# #=> 0.325
|
36
61
|
def self.write units, opts = {}
|
37
|
-
pricing = HOURLY_PRICING[
|
38
|
-
|
62
|
+
if pricing = HOURLY_PRICING[AWS.config.region]
|
63
|
+
((units / pricing[:write][:per].to_f) * pricing[:write][:dollars])
|
64
|
+
else
|
65
|
+
nil
|
66
|
+
end
|
39
67
|
end
|
40
68
|
end
|
41
69
|
end
|
data/script/historic_data
CHANGED
@@ -37,7 +37,7 @@ DynamoAutoscale.poller.tables.select! do |table|
|
|
37
37
|
end
|
38
38
|
|
39
39
|
range.each do |start_day|
|
40
|
-
dir =
|
40
|
+
dir = DynamoAutoscale.data_dir(start_day.to_s)
|
41
41
|
end_day = start_day + 1.day
|
42
42
|
|
43
43
|
FileUtils.mkdir(dir) unless Dir.exists?(dir)
|