dynamo-autoscale 0.3.6 → 0.4.1

data/lib/dynamo-autoscale/fake_poller.rb

@@ -0,0 +1,40 @@
+module DynamoAutoscale
+  # This class allows you to specify exactly what data to send through the
+  # poller.
+  #
+  # Example:
+  #
+  #   DynamoAutoscale.poller_class = FakePoller
+  #   DynamoAutoscale.poller_opts  = {
+  #     data: {
+  #       Time.now => {
+  #         provisioned_reads:  100,
+  #         provisioned_writes: 100,
+  #         consumed_reads:     55.6,
+  #         consumed_writes:    12.7,
+  #       },
+  #       Time.now + 15.minutes => {
+  #         provisioned_reads:  100,
+  #         provisioned_writes: 100,
+  #         consumed_reads:     45.9,
+  #         consumed_writes:    7.1,
+  #       },
+  #     }
+  #   }
+  class FakePoller < Poller
+    def initialize *args
+      super(*args)
+      opts = args.last.is_a?(Hash) ? args.last : {}
+
+      @data = opts[:data]
+    end
+
+    def poll tables, &block
+      @data.each do |key, value|
+        tables.each do |table_name|
+          block.call(table_name, key => value)
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-autoscale/log_collector.rb

@@ -0,0 +1,18 @@
+module DynamoAutoscale
+  class LogCollector < ::Logger
+    attr_reader :messages
+
+    def initialize log_to
+      super
+
+      @messages = Hash.new { |h, k| h[k] = [] }
+    end
+
+    [:debug, :info, :warn, :error, :fatal].each do |level|
+      define_method level do |message|
+        @messages[level] << message
+        super message
+      end
+    end
+  end
+end

data/lib/dynamo-autoscale/logger.rb

@@ -5,13 +5,17 @@ module DynamoAutoscale
     end
 
     def self.logger
-      @@logger
+      @@logger ||= ::Logger.new(STDOUT)
     end
 
     def logger
       DynamoAutoscale::Logger.logger
     end
 
+    def logger= new_logger
+      DynamoAutoscale::Logger.logger = new_logger
+    end
+
     def self.included base
       base.extend DynamoAutoscale::Logger
     end

data/lib/dynamo-autoscale/metrics.rb

@@ -3,32 +3,10 @@ module DynamoAutoscale
     include DynamoAutoscale::Logger
 
     DEFAULT_OPTS = {
-      namespace:   'AWS/DynamoDB',
-      period:      300,
-      # metric_name: metric,
-      # start_time:  (NOW - 3600).iso8601,
-      # end_time:    NOW.iso8601,
-      # dimensions:  [{
-      #   name: "TableName", value: TABLE_NAME,
-      # }],
+      namespace: 'AWS/DynamoDB',
+      period:    300,
     }
 
-    # Returns a CloudWatch client object for a given region. If no region
-    # exists, the region defaults to whatever is in
-    # DynamoAutoscale::DEFAULT_AWS_REGION.
-    #
-    # CloudWatch client documentation:
-    #   https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/cloud_watch/client.rb
-    def self.client region = nil
-      @client ||= Hash.new do |hash, _region|
-        hash[_region] = AWS::CloudWatch.new({
-          cloud_watch_endpoint: "monitoring.#{_region}.amazonaws.com",
-        }).client
-      end
-
-      @client[region || DEFAULT_AWS_REGION]
-    end
-
     # Returns a hash of timeseries data. Looks a bit like this:
     #
     #   {
@@ -171,22 +149,27 @@ module DynamoAutoscale
     # A base method that gets called by wrapper methods defined above. Makes a
     # call to CloudWatch, getting statistics on whatever metric is given.
     def self.metric_statistics table_name, opts = {}
-      region = opts.delete :region
       opts   = DEFAULT_OPTS.merge({
         dimensions:  [{ name: "TableName", value: table_name }],
         start_time:  1.hour.ago,
         end_time:    Time.now,
       }).merge(opts)
 
-      if opts[:start_time] and opts[:start_time].respond_to? :iso8601
+      if opts[:start_time] and opts[:start_time].respond_to?(:iso8601)
         opts[:start_time] = opts[:start_time].iso8601
       end
 
-      if opts[:end_time] and opts[:end_time].respond_to? :iso8601
+      if opts[:end_time] and opts[:end_time].respond_to?(:iso8601)
         opts[:end_time] = opts[:end_time].iso8601
       end
 
-      client(region).get_metric_statistics(opts)[:datapoints]
+      client.get_metric_statistics(opts)[:datapoints]
+    end
+
+    private
+
+    def self.client
+      @@client ||= AWS::CloudWatch.new.client
     end
   end
 end

data/lib/dynamo-autoscale/poller.rb

@@ -30,12 +30,23 @@ module DynamoAutoscale
       end.sort!.uniq
 
       times.each do |time|
-        datum = {
-          provisioned_writes: data[:provisioned_writes][time],
-          provisioned_reads:  data[:provisioned_reads][time],
-          consumed_writes:    data[:consumed_writes][time],
-          consumed_reads:     data[:consumed_reads][time],
-        }
+        datum = {}
+
+        if data[:provisioned_writes] and data[:provisioned_writes][time]
+          datum[:provisioned_writes] = data[:provisioned_writes][time]
+        end
+
+        if data[:provisioned_reads] and data[:provisioned_reads][time]
+          datum[:provisioned_reads] = data[:provisioned_reads][time]
+        end
+
+        if data[:consumed_writes] and data[:consumed_writes][time]
+          datum[:consumed_writes] = data[:consumed_writes][time]
+        end
+
+        if data[:consumed_reads] and data[:consumed_reads][time]
+          datum[:consumed_reads] = data[:consumed_reads][time]
+        end
 
         filters.each { |filter| filter.call(table, time, datum) }
 

data/lib/dynamo-autoscale/random_data_generator.rb

@@ -0,0 +1,51 @@
+module DynamoAutoscale
+  # You can use this class as the DynamoAutoscale.poller global and it will just
+  # generate a given number of random data points at 15 minutes intervals. Nice
+  # for testing purposes.
+  class RandomDataGenerator < Poller
+    def initialize *args
+      super(*args)
+      opts = args.last.is_a?(Hash) ? args.last : {}
+
+      @num_points         = opts[:num_points] || 100
+      @start_time         = opts[:start_time] || Time.now
+      @current_time       = @start_time
+      @provisioned_reads  = opts[:provisioned_reads] || 600
+      @provisioned_writes = opts[:provisioned_writes] || 600
+
+      srand(@start_time.to_i)
+    end
+
+    def poll tables, &block
+      # Give each table its initial provisioned reads and writes before starting
+      # the random data generation.
+      tables.each do |table_name|
+        block.call(table_name, {
+          provisioned_reads: {
+            @start_time => @provisioned_reads,
+          },
+          provisioned_writes: {
+            @start_time => @provisioned_writes,
+          },
+        })
+      end
+
+      @num_points.times do
+        tables.each do |table_name|
+          # Give each table varying figures for consumed reads and writes that
+          # hover between 0 and their provisioned values multiplied by 2.
+          block.call(table_name, {
+            consumed_reads: {
+              @current_time => rand * @provisioned_reads * 2,
+            },
+            consumed_writes: {
+              @current_time => rand * @provisioned_writes * 2,
+            },
+          })
+
+          @current_time += 15.minutes
+        end
+      end
+    end
+  end
+end

data/lib/dynamo-autoscale/scale_report.rb

@@ -2,7 +2,7 @@ module DynamoAutoscale
   class ScaleReport
     include DynamoAutoscale::Logger
 
-    TEMPLATE = File.join(DynamoAutoscale.root, 'templates', 'scale_report_email.erb')
+    TEMPLATE = DynamoAutoscale.template_dir('scale_report_email.erb')
 
     def initialize table
       @table = table
@@ -48,7 +48,7 @@ module DynamoAutoscale
 
     def formatted_scale_event(scale_event)
       max_length = max_metric_length(scale_event)
-      
+
       ['reads', 'writes'].map do |type|
         next unless scale_event.has_key? "#{type}_from".to_sym
 

data/lib/dynamo-autoscale/table_tracker.rb

@@ -198,6 +198,30 @@ module DynamoAutoscale
       end
     end
 
+    def wasted_write_cost
+      UnitCost.write(wasted_write_units)
+    end
+
+    def wasted_read_cost
+      UnitCost.read(wasted_read_units)
+    end
+
+    def lost_write_cost
+      UnitCost.write(lost_write_units)
+    end
+
+    def lost_read_cost
+      UnitCost.read(lost_read_units)
+    end
+
+    def total_write_cost
+      UnitCost.write(total_write_units)
+    end
+
+    def total_read_cost
+      UnitCost.read(total_read_units)
+    end
+
     def wasted_read_percent
       (wasted_read_units / total_read_units) * 100.0
     end
@@ -243,7 +267,7 @@ module DynamoAutoscale
     # end
 
     def to_csv! opts = {}
-      path = opts[:path] or File.join(DynamoAutoscale.root, "#{self.name}.csv")
+      path = opts[:path] or DynamoAutoscale.root_dir("#{self.name}.csv")
 
       CSV.open(path, 'w') do |csv|
         csv << [
@@ -271,7 +295,7 @@ module DynamoAutoscale
     def graph! opts = {}
       data_tmp = File.join(Dir.tmpdir, 'data.csv')
       png_tmp  = opts[:path] || File.join(Dir.tmpdir, 'graph.png')
-      r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_graph.r')
+      r_script = DynamoAutoscale.rlib_dir('dynamodb_graph.r')
 
       to_csv!(path: data_tmp)
 
@@ -296,7 +320,7 @@ module DynamoAutoscale
     def scatterplot_for! metric
       data_tmp = File.join(Dir.tmpdir, 'data.csv')
       png_tmp  = File.join(Dir.tmpdir, 'boxplot.png')
-      r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_boxplot.r')
+      r_script = DynamoAutoscale.rlib_dir('dynamodb_boxplot.r')
 
       to_csv!(data_tmp)
 
@@ -309,16 +333,35 @@ module DynamoAutoscale
       end
     end
 
-    def report!
-      puts "         Table: #{name}"
-      puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
-      puts " Total r/units: #{total_read_units.round(2)}"
-      puts "  Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
-      puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
-      puts " Total w/units: #{total_write_units.round(2)}"
-      puts "  Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
-      puts "      Upscales: #{DynamoAutoscale.actioners[self].upscales}"
-      puts "    Downscales: #{DynamoAutoscale.actioners[self].downscales}"
+    def report! opts = {}
+      opts[:metric] ||= :units
+
+      unless [:cost, :units].include?(opts[:metric])
+        raise ArgumentError.new("The :metric option must be one of :cost or " +
+          ":units")
+      end
+
+      if opts[:metric] == :units
+        puts "         Table: #{name}"
+        puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
+        puts " Total r/units: #{total_read_units.round(2)}"
+        puts "  Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
+        puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
+        puts " Total w/units: #{total_write_units.round(2)}"
+        puts "  Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
+        puts "      Upscales: #{DynamoAutoscale.actioners[self].upscales}"
+        puts "    Downscales: #{DynamoAutoscale.actioners[self].downscales}"
+      elsif opts[:metric] == :cost
+        puts "         Table: #{name}"
+        puts "Wasted r/cost: $#{wasted_read_cost.round(2)} (#{wasted_read_percent.round(2)}%)"
+        puts " Total r/cost: $#{total_read_cost.round(2)}"
+        puts "  Lost r/cost: $#{lost_read_cost.round(2)} (#{lost_read_percent.round(2)}%)"
+        puts "Wasted w/cost: $#{wasted_write_cost.round(2)} (#{wasted_write_percent.round(2)}%)"
+        puts " Total w/cost: $#{total_write_cost.round(2)}"
+        puts "  Lost w/cost: $#{lost_write_cost.round(2)} (#{lost_write_percent.round(2)}%)"
+        puts "      Upscales: #{DynamoAutoscale.actioners[self].upscales}"
+        puts "    Downscales: #{DynamoAutoscale.actioners[self].downscales}"
+      end
     end
 
     private

data/lib/dynamo-autoscale/unit_cost.rb

@@ -2,40 +2,68 @@ module DynamoAutoscale
   class UnitCost
     # Pricing information obtained from: http://aws.amazon.com/dynamodb/pricing/
     HOURLY_PRICING = {
-      'us-east-1' => {
+      'us-east-1' => { # US East (Northern Virginia)
         read:  { dollars: 0.0065, per: 50 },
         write: { dollars: 0.0065, per: 10 },
       },
-      'us-west-1' => {
+      'us-west-1' => { # US West (Northern California)
         read:  { dollars: 0.0065, per: 50 },
         write: { dollars: 0.0065, per: 10 },
       },
+      'us-west-2' => { # US West (Oregon)
+        read:  { dollars: 0.0065, per: 50 },
+        write: { dollars: 0.0065, per: 10 },
+      },
+      'ap-southeast-1' => { # Asia Pacific (Singapore)
+        read:  { dollars: 0.0074, per: 50 },
+        write: { dollars: 0.0074, per: 10 },
+      },
+      'ap-southeast-2' => { # Asia Pacific (Sydney)
+        read:  { dollars: 0.0074, per: 50 },
+        write: { dollars: 0.0074, per: 10 },
+      },
+      'ap-northeast-1' => { # Asia Pacific (Tokyo)
+        read:  { dollars: 0.0078, per: 50 },
+        write: { dollars: 0.0078, per: 10 },
+      },
+      'eu-west-1' => { # EU (Ireland)
+        read:  { dollars: 0.00735, per: 50 },
+        write: { dollars: 0.00735, per: 10 },
+      },
+      'sa-east-1' => { # South America (Sao Paulo)
+        read:  { dollars: 0.00975, per: 50 },
+        write: { dollars: 0.00975, per: 10 },
+      },
     }
 
-    # Returns the cost of N read units for an hour in a given region, which
-    # defaults to whatever is in
-    # DynamoAutoscale::DEFAULT_AWS_REGION.
+    # Returns the cost of N read units for an hour in the region given by
+    # AWS.config.region
     #
     # Example:
     #
-    #   DynamoAutoscale::UnitCost.read(500, region: 'us-west-1')
+    #   DynamoAutoscale::UnitCost.read(500)
     #   #=> 0.065
     def self.read units, opts = {}
-      pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:read]
-      ((units / pricing[:per].to_f) * pricing[:dollars])
+      if pricing = HOURLY_PRICING[AWS.config.region]
+        ((units / pricing[:read][:per].to_f) * pricing[:read][:dollars])
+      else
+        nil
+      end
     end
 
-    # Returns the cost of N write units for an hour in a given region, which
-    # defaults to whatever is in
-    # DynamoAutoscale::DEFAULT_AWS_REGION.
+    # Returns the cost of N write units for an hour in the region given by
+    # AWS.config.region.
     #
     # Example:
     #
-    #   DynamoAutoscale::UnitCost.write(500, region: 'us-west-1')
+    #   DynamoAutoscale::UnitCost.write(500)
     #   #=> 0.325
     def self.write units, opts = {}
-      pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:write]
-      ((units / pricing[:per].to_f) * pricing[:dollars])
+      if pricing = HOURLY_PRICING[AWS.config.region]
+        ((units / pricing[:write][:per].to_f) * pricing[:write][:dollars])
+      else
+        nil
+      end
     end
   end
 end

data/lib/dynamo-autoscale/version.rb

@@ -1,3 +1,3 @@
 module DynamoAutoscale
-  VERSION = '0.3.6'
+  VERSION = '0.4.1'
 end

data/script/historic_data

@@ -37,7 +37,7 @@ DynamoAutoscale.poller.tables.select! do |table|
 end
 
 range.each do |start_day|
-  dir     = File.join(DynamoAutoscale.data_dir, start_day.to_s)
+  dir     = DynamoAutoscale.data_dir(start_day.to_s)
   end_day = start_day + 1.day
 
   FileUtils.mkdir(dir) unless Dir.exists?(dir)