dynamo-autoscale 0.3.6 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,40 @@
1
+ module DynamoAutoscale
2
+ # This class allows you to specify exactly what data to send through the
3
+ # poller.
4
+ #
5
+ # Example:
6
+ #
7
+ # DynamoAutoscale.poller_class = FakePoller
8
+ # DynamoAutoscale.poller_opts = {
9
+ # data: {
10
+ # Time.now => {
11
+ # provisioned_reads: 100,
12
+ # provisioned_writes: 100,
13
+ # consumed_reads: 55.6,
14
+ # consumed_writes: 12.7,
15
+ # },
16
+ # Time.now + 15.minutes => {
17
+ # provisioned_reads: 100,
18
+ # provisioned_writes: 100,
19
+ # consumed_reads: 45.9,
20
+ # consumed_writes: 7.1,
21
+ # },
22
+ # }
23
+ # }
24
+ class FakePoller < Poller
25
+ def initialize *args
26
+ super(*args)
27
+ opts = args.last.is_a?(Hash) ? args.last : {}
28
+
29
+ @data = opts[:data]
30
+ end
31
+
32
+ def poll tables, &block
33
+ @data.each do |key, value|
34
+ tables.each do |table_name|
35
+ block.call(table_name, key => value)
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,18 @@
1
+ module DynamoAutoscale
2
+ class LogCollector < ::Logger
3
+ attr_reader :messages
4
+
5
+ def initialize log_to
6
+ super
7
+
8
+ @messages = Hash.new { |h, k| h[k] = [] }
9
+ end
10
+
11
+ [:debug, :info, :warn, :error, :fatal].each do |level|
12
+ define_method level do |message|
13
+ @messages[level] << message
14
+ super message
15
+ end
16
+ end
17
+ end
18
+ end
@@ -5,13 +5,17 @@ module DynamoAutoscale
5
5
  end
6
6
 
7
7
  def self.logger
8
- @@logger
8
+ @@logger ||= ::Logger.new(STDOUT)
9
9
  end
10
10
 
11
11
  def logger
12
12
  DynamoAutoscale::Logger.logger
13
13
  end
14
14
 
15
+ def logger= new_logger
16
+ DynamoAutoscale::Logger.logger = new_logger
17
+ end
18
+
15
19
  def self.included base
16
20
  base.extend DynamoAutoscale::Logger
17
21
  end
@@ -3,32 +3,10 @@ module DynamoAutoscale
3
3
  include DynamoAutoscale::Logger
4
4
 
5
5
  DEFAULT_OPTS = {
6
- namespace: 'AWS/DynamoDB',
7
- period: 300,
8
- # metric_name: metric,
9
- # start_time: (NOW - 3600).iso8601,
10
- # end_time: NOW.iso8601,
11
- # dimensions: [{
12
- # name: "TableName", value: TABLE_NAME,
13
- # }],
6
+ namespace: 'AWS/DynamoDB',
7
+ period: 300,
14
8
  }
15
9
 
16
- # Returns a CloudWatch client object for a given region. If no region
17
- # exists, the region defaults to whatever is in
18
- # DynamoAutoscale::DEFAULT_AWS_REGION.
19
- #
20
- # CloudWatch client documentation:
21
- # https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/cloud_watch/client.rb
22
- def self.client region = nil
23
- @client ||= Hash.new do |hash, _region|
24
- hash[_region] = AWS::CloudWatch.new({
25
- cloud_watch_endpoint: "monitoring.#{_region}.amazonaws.com",
26
- }).client
27
- end
28
-
29
- @client[region || DEFAULT_AWS_REGION]
30
- end
31
-
32
10
  # Returns a hash of timeseries data. Looks a bit like this:
33
11
  #
34
12
  # {
@@ -171,22 +149,27 @@ module DynamoAutoscale
171
149
  # A base method that gets called by wrapper methods defined above. Makes a
172
150
  # call to CloudWatch, getting statistics on whatever metric is given.
173
151
  def self.metric_statistics table_name, opts = {}
174
- region = opts.delete :region
175
152
  opts = DEFAULT_OPTS.merge({
176
153
  dimensions: [{ name: "TableName", value: table_name }],
177
154
  start_time: 1.hour.ago,
178
155
  end_time: Time.now,
179
156
  }).merge(opts)
180
157
 
181
- if opts[:start_time] and opts[:start_time].respond_to? :iso8601
158
+ if opts[:start_time] and opts[:start_time].respond_to?(:iso8601)
182
159
  opts[:start_time] = opts[:start_time].iso8601
183
160
  end
184
161
 
185
- if opts[:end_time] and opts[:end_time].respond_to? :iso8601
162
+ if opts[:end_time] and opts[:end_time].respond_to?(:iso8601)
186
163
  opts[:end_time] = opts[:end_time].iso8601
187
164
  end
188
165
 
189
- client(region).get_metric_statistics(opts)[:datapoints]
166
+ client.get_metric_statistics(opts)[:datapoints]
167
+ end
168
+
169
+ private
170
+
171
+ def self.client
172
+ @@client ||= AWS::CloudWatch.new.client
190
173
  end
191
174
  end
192
175
  end
@@ -30,12 +30,23 @@ module DynamoAutoscale
30
30
  end.sort!.uniq
31
31
 
32
32
  times.each do |time|
33
- datum = {
34
- provisioned_writes: data[:provisioned_writes][time],
35
- provisioned_reads: data[:provisioned_reads][time],
36
- consumed_writes: data[:consumed_writes][time],
37
- consumed_reads: data[:consumed_reads][time],
38
- }
33
+ datum = {}
34
+
35
+ if data[:provisioned_writes] and data[:provisioned_writes][time]
36
+ datum[:provisioned_writes] = data[:provisioned_writes][time]
37
+ end
38
+
39
+ if data[:provisioned_reads] and data[:provisioned_reads][time]
40
+ datum[:provisioned_reads] = data[:provisioned_reads][time]
41
+ end
42
+
43
+ if data[:consumed_writes] and data[:consumed_writes][time]
44
+ datum[:consumed_writes] = data[:consumed_writes][time]
45
+ end
46
+
47
+ if data[:consumed_reads] and data[:consumed_reads][time]
48
+ datum[:consumed_reads] = data[:consumed_reads][time]
49
+ end
39
50
 
40
51
  filters.each { |filter| filter.call(table, time, datum) }
41
52
 
@@ -0,0 +1,51 @@
1
+ module DynamoAutoscale
2
+ # You can use this class as the DynamoAutoscale.poller global and it will just
3
+ # generate a given number of random data points at 15 minutes intervals. Nice
4
+ # for testing purposes.
5
+ class RandomDataGenerator < Poller
6
+ def initialize *args
7
+ super(*args)
8
+ opts = args.last.is_a?(Hash) ? args.last : {}
9
+
10
+ @num_points = opts[:num_points] || 100
11
+ @start_time = opts[:start_time] || Time.now
12
+ @current_time = @start_time
13
+ @provisioned_reads = opts[:provisioned_reads] || 600
14
+ @provisioned_writes = opts[:provisioned_writes] || 600
15
+
16
+ srand(@start_time.to_i)
17
+ end
18
+
19
+ def poll tables, &block
20
+ # Give each table its initial provisioned reads and writes before starting
21
+ # the random data generation.
22
+ tables.each do |table_name|
23
+ block.call(table_name, {
24
+ provisioned_reads: {
25
+ @start_time => @provisioned_reads,
26
+ },
27
+ provisioned_writes: {
28
+ @start_time => @provisioned_writes,
29
+ },
30
+ })
31
+ end
32
+
33
+ @num_points.times do
34
+ tables.each do |table_name|
35
+ # Give each table varying figures for consumed reads and writes that
36
+ # hover between 0 and their provisioned values multiplied by 2.
37
+ block.call(table_name, {
38
+ consumed_reads: {
39
+ @current_time => rand * @provisioned_reads * 2,
40
+ },
41
+ consumed_writes: {
42
+ @current_time => rand * @provisioned_writes * 2,
43
+ },
44
+ })
45
+
46
+ @current_time += 15.minutes
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -2,7 +2,7 @@ module DynamoAutoscale
2
2
  class ScaleReport
3
3
  include DynamoAutoscale::Logger
4
4
 
5
- TEMPLATE = File.join(DynamoAutoscale.root, 'templates', 'scale_report_email.erb')
5
+ TEMPLATE = DynamoAutoscale.template_dir('scale_report_email.erb')
6
6
 
7
7
  def initialize table
8
8
  @table = table
@@ -48,7 +48,7 @@ module DynamoAutoscale
48
48
 
49
49
  def formatted_scale_event(scale_event)
50
50
  max_length = max_metric_length(scale_event)
51
-
51
+
52
52
  ['reads', 'writes'].map do |type|
53
53
  next unless scale_event.has_key? "#{type}_from".to_sym
54
54
 
@@ -198,6 +198,30 @@ module DynamoAutoscale
198
198
  end
199
199
  end
200
200
 
201
+ def wasted_write_cost
202
+ UnitCost.write(wasted_write_units)
203
+ end
204
+
205
+ def wasted_read_cost
206
+ UnitCost.read(wasted_read_units)
207
+ end
208
+
209
+ def lost_write_cost
210
+ UnitCost.write(lost_write_units)
211
+ end
212
+
213
+ def lost_read_cost
214
+ UnitCost.read(lost_read_units)
215
+ end
216
+
217
+ def total_write_cost
218
+ UnitCost.write(total_write_units)
219
+ end
220
+
221
+ def total_read_cost
222
+ UnitCost.read(total_read_units)
223
+ end
224
+
201
225
  def wasted_read_percent
202
226
  (wasted_read_units / total_read_units) * 100.0
203
227
  end
@@ -243,7 +267,7 @@ module DynamoAutoscale
243
267
  # end
244
268
 
245
269
  def to_csv! opts = {}
246
- path = opts[:path] or File.join(DynamoAutoscale.root, "#{self.name}.csv")
270
+ path = opts[:path] or DynamoAutoscale.root_dir("#{self.name}.csv")
247
271
 
248
272
  CSV.open(path, 'w') do |csv|
249
273
  csv << [
@@ -271,7 +295,7 @@ module DynamoAutoscale
271
295
  def graph! opts = {}
272
296
  data_tmp = File.join(Dir.tmpdir, 'data.csv')
273
297
  png_tmp = opts[:path] || File.join(Dir.tmpdir, 'graph.png')
274
- r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_graph.r')
298
+ r_script = DynamoAutoscale.rlib_dir('dynamodb_graph.r')
275
299
 
276
300
  to_csv!(path: data_tmp)
277
301
 
@@ -296,7 +320,7 @@ module DynamoAutoscale
296
320
  def scatterplot_for! metric
297
321
  data_tmp = File.join(Dir.tmpdir, 'data.csv')
298
322
  png_tmp = File.join(Dir.tmpdir, 'boxplot.png')
299
- r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_boxplot.r')
323
+ r_script = DynamoAutoscale.rlib_dir('dynamodb_boxplot.r')
300
324
 
301
325
  to_csv!(data_tmp)
302
326
 
@@ -309,16 +333,35 @@ module DynamoAutoscale
309
333
  end
310
334
  end
311
335
 
312
- def report!
313
- puts " Table: #{name}"
314
- puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
315
- puts " Total r/units: #{total_read_units.round(2)}"
316
- puts " Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
317
- puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
318
- puts " Total w/units: #{total_write_units.round(2)}"
319
- puts " Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
320
- puts " Upscales: #{DynamoAutoscale.actioners[self].upscales}"
321
- puts " Downscales: #{DynamoAutoscale.actioners[self].downscales}"
336
+ def report! opts = {}
337
+ opts[:metric] ||= :units
338
+
339
+ unless [:cost, :units].include?(opts[:metric])
340
+ raise ArgumentError.new("The :metric option must be one of :cost or " +
341
+ ":units")
342
+ end
343
+
344
+ if opts[:metric] == :units
345
+ puts " Table: #{name}"
346
+ puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
347
+ puts " Total r/units: #{total_read_units.round(2)}"
348
+ puts " Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
349
+ puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
350
+ puts " Total w/units: #{total_write_units.round(2)}"
351
+ puts " Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
352
+ puts " Upscales: #{DynamoAutoscale.actioners[self].upscales}"
353
+ puts " Downscales: #{DynamoAutoscale.actioners[self].downscales}"
354
+ elsif opts[:metric] == :cost
355
+ puts " Table: #{name}"
356
+ puts "Wasted r/cost: $#{wasted_read_cost.round(2)} (#{wasted_read_percent.round(2)}%)"
357
+ puts " Total r/cost: $#{total_read_cost.round(2)}"
358
+ puts " Lost r/cost: $#{lost_read_cost.round(2)} (#{lost_read_percent.round(2)}%)"
359
+ puts "Wasted w/cost: $#{wasted_write_cost.round(2)} (#{wasted_write_percent.round(2)}%)"
360
+ puts " Total w/cost: $#{total_write_cost.round(2)}"
361
+ puts " Lost w/cost: $#{lost_write_cost.round(2)} (#{lost_write_percent.round(2)}%)"
362
+ puts " Upscales: #{DynamoAutoscale.actioners[self].upscales}"
363
+ puts " Downscales: #{DynamoAutoscale.actioners[self].downscales}"
364
+ end
322
365
  end
323
366
 
324
367
  private
@@ -2,40 +2,68 @@ module DynamoAutoscale
2
2
  class UnitCost
3
3
  # Pricing information obtained from: http://aws.amazon.com/dynamodb/pricing/
4
4
  HOURLY_PRICING = {
5
- 'us-east-1' => {
5
+ 'us-east-1' => { # US East (Northern Virginia)
6
6
  read: { dollars: 0.0065, per: 50 },
7
7
  write: { dollars: 0.0065, per: 10 },
8
8
  },
9
- 'us-west-1' => {
9
+ 'us-west-1' => { # US West (Northern California)
10
10
  read: { dollars: 0.0065, per: 50 },
11
11
  write: { dollars: 0.0065, per: 10 },
12
12
  },
13
+ 'us-west-2' => { # US West (Oregon)
14
+ read: { dollars: 0.0065, per: 50 },
15
+ write: { dollars: 0.0065, per: 10 },
16
+ },
17
+ 'ap-southeast-1' => { # Asia Pacific (Singapore)
18
+ read: { dollars: 0.0074, per: 50 },
19
+ write: { dollars: 0.0074, per: 10 },
20
+ },
21
+ 'ap-southeast-2' => { # Asia Pacific (Sydney)
22
+ read: { dollars: 0.0074, per: 50 },
23
+ write: { dollars: 0.0074, per: 10 },
24
+ },
25
+ 'ap-northeast-1' => { # Asia Pacific (Tokyo)
26
+ read: { dollars: 0.0078, per: 50 },
27
+ write: { dollars: 0.0078, per: 10 },
28
+ },
29
+ 'eu-west-1' => { # EU (Ireland)
30
+ read: { dollars: 0.00735, per: 50 },
31
+ write: { dollars: 0.00735, per: 10 },
32
+ },
33
+ 'sa-east-1' => { # South America (Sao Paulo)
34
+ read: { dollars: 0.00975, per: 50 },
35
+ write: { dollars: 0.00975, per: 10 },
36
+ },
13
37
  }
14
38
 
15
- # Returns the cost of N read units for an hour in a given region, which
16
- # defaults to whatever is in
17
- # DynamoAutoscale::DEFAULT_AWS_REGION.
39
+ # Returns the cost of N read units for an hour in the region given by
40
+ # AWS.config.region
18
41
  #
19
42
  # Example:
20
43
  #
21
- # DynamoAutoscale::UnitCost.read(500, region: 'us-west-1')
44
+ # DynamoAutoscale::UnitCost.read(500)
22
45
  # #=> 0.065
23
46
  def self.read units, opts = {}
24
- pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:read]
25
- ((units / pricing[:per].to_f) * pricing[:dollars])
47
+ if pricing = HOURLY_PRICING[AWS.config.region]
48
+ ((units / pricing[:read][:per].to_f) * pricing[:read][:dollars])
49
+ else
50
+ nil
51
+ end
26
52
  end
27
53
 
28
- # Returns the cost of N write units for an hour in a given region, which
29
- # defaults to whatever is in
30
- # DynamoAutoscale::DEFAULT_AWS_REGION.
54
+ # Returns the cost of N write units for an hour in the region given by
55
+ # AWS.config.region.
31
56
  #
32
57
  # Example:
33
58
  #
34
- # DynamoAutoscale::UnitCost.write(500, region: 'us-west-1')
59
+ # DynamoAutoscale::UnitCost.write(500)
35
60
  # #=> 0.325
36
61
  def self.write units, opts = {}
37
- pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:write]
38
- ((units / pricing[:per].to_f) * pricing[:dollars])
62
+ if pricing = HOURLY_PRICING[AWS.config.region]
63
+ ((units / pricing[:write][:per].to_f) * pricing[:write][:dollars])
64
+ else
65
+ nil
66
+ end
39
67
  end
40
68
  end
41
69
  end
@@ -1,3 +1,3 @@
1
1
  module DynamoAutoscale
2
- VERSION = '0.3.6'
2
+ VERSION = '0.4.1'
3
3
  end
@@ -37,7 +37,7 @@ DynamoAutoscale.poller.tables.select! do |table|
37
37
  end
38
38
 
39
39
  range.each do |start_day|
40
- dir = File.join(DynamoAutoscale.data_dir, start_day.to_s)
40
+ dir = DynamoAutoscale.data_dir(start_day.to_s)
41
41
  end_day = start_day + 1.day
42
42
 
43
43
  FileUtils.mkdir(dir) unless Dir.exists?(dir)