dynamo-autoscale 0.1

data/lib/dynamo-autoscale/table_tracker.rb

@@ -0,0 +1,329 @@
+module DynamoAutoscale
+  class TableTracker
+    include DynamoAutoscale::Logger
+
+    # TODO: This time window may need changing.
+    TIME_WINDOW = 7.days
+
+    attr_reader :name, :data
+
+    def initialize name
+      @name = name
+      clear_data
+    end
+
+    def clear_data
+      @data = RBTree.new
+    end
+
+    # `tick` takes two arguments. The first is a Time object, the second is
+    # a hash. The tick method expects data in the following format for the
+    # second argument:
+    #
+    #   {:provisioned_writes=>600.0,
+    #    :provisioned_reads=>800.0,
+    #    :consumed_writes=>52.693333333333335,
+    #    :consumed_reads=>342.4033333333333}
+    def tick time, datum
+      if time < (Time.now.utc - TIME_WINDOW)
+        logger.warn "Attempted to insert data outside of the time window."
+        return
+      end
+
+      # Sometimes there are gaps in the data pertaining to provisioned
+      # amounts. These two conditional blocks fill in those gaps.
+      if datum[:provisioned_writes].nil?
+        datum[:provisioned_writes] = last_provisioned_for :writes, at: time
+
+        if datum[:provisioned_writes]
+          logger.debug "Filled in gap in provisioned writes."
+        end
+      end
+      if datum[:provisioned_reads].nil?
+        datum[:provisioned_reads] = last_provisioned_for :reads, at: time
+
+        if datum[:provisioned_reads]
+          logger.debug "Filled in gap in provisioned reads."
+        end
+      end
+
+      @data[time] = datum
+
+      # The code below here just makes sure that we're trimming data points that
+      # are outside of the time window.
+      logger.debug "Pruning data that may be outside of time window..."
+      now = Time.now.utc
+      to_delete = @data.each.take_while { |key, _| key < (now - TIME_WINDOW) }
+      to_delete.each { |key, _| @data.delete(key) }
+    end
+
+    # Gets the last amount of provisioned throughput for whatever metric you
+    # pass in. Example:
+    #
+    #   table.last_provisioned_for :writes
+    #   #=> 600.0
+    def last_provisioned_for metric, opts = {}
+      key = case metric
+      when :reads, :provisioned_reads, :consumed_reads
+        :provisioned_reads
+      when :writes, :provisioned_writes, :consumed_writes
+        :provisioned_writes
+      end
+
+      @data.reverse_each do |time, datum|
+        if opts[:at].nil? or time <= opts[:at]
+          return datum[key] if datum[key]
+        end
+      end
+
+      return nil
+    end
+
+    # Gets the last amount of consumed throughput for whatever metric you
+    # pass in. Example:
+    #
+    #   table.last_consumed_for :writes
+    #   #=> 54.3456
+    def last_consumed_for metric, opts = {}
+      key = case metric
+      when :reads, :provisioned_reads, :consumed_reads
+        :consumed_reads
+      when :writes, :provisioned_writes, :consumed_writes
+        :consumed_writes
+      end
+
+      @data.reverse_each do |time, datum|
+        if opts[:at].nil? or time <= opts[:at]
+          return datum[key] if datum[key]
+        end
+      end
+
+      return nil
+    end
+
+    # Useful method for querying the last N points, or the last points in a
+    # time range. For example:
+    #
+    #    table.last 5. :consumed_writes
+    #    #=> [ array of last 5 data points ]
+    #
+    #    table.last 5.minutes, :provisioned_reads
+    #    #=> [ array containing the last 5 minutes of provisioned read data ]
+    #
+    # If there are no points present, or no points in your time range, the
+    # return value will be an empty array.
+    def last value, metric
+      if value.is_a? ActiveSupport::Duration
+        value = value.to_i
+        to_return = []
+        now = Time.now.to_i
+
+        @data.reverse_each do |time, datum|
+          value -= now - time.to_i
+          now    = time.to_i
+          break if value < 0
+
+          to_return << datum[metric]
+        end
+
+        to_return
+      else
+        @data.reverse_each.take(value).map { |time, datum| datum[metric] }
+      end
+    end
+
+    # Calculate how many read units have been wasted in the current set of
+    # tracked data.
+    #
+    #   table.wasted_read_units
+    #   #=> 244.4
+    def wasted_read_units
+      @data.inject(0) do |memo, (_, datum)|
+        # if datum[:provisioned_reads] and datum[:consumed_reads]
+          memo += datum[:provisioned_reads] - datum[:consumed_reads]
+        # end
+
+        memo
+      end
+    end
+
+    # Calculate how many write units have been wasted in the current set of
+    # tracked data.
+    #
+    #   table.wasted_write_units
+    #   #=> 566.3
+    def wasted_write_units
+      @data.inject(0) do |memo, (_, datum)|
+        # if datum[:provisioned_writes] and datum[:consumed_writes]
+          memo += datum[:provisioned_writes] - datum[:consumed_writes]
+        # end
+
+        memo
+      end
+    end
+
+    # Whenever the consumed units goes above the provisioned, we refer to the
+    # overflow as "lost" units.
+    def lost_read_units
+      @data.inject(0) do |memo, (_, datum)|
+        if datum[:consumed_reads] > datum[:provisioned_reads]
+          memo += datum[:consumed_reads] - datum[:provisioned_reads]
+        end
+
+        memo
+      end
+    end
+
+    # Whenever the consumed units goes above the provisioned, we refer to the
+    # overflow as "lost" units.
+    def lost_write_units
+      @data.inject(0) do |memo, (_, datum)|
+        if datum[:consumed_writes] > datum[:provisioned_writes]
+          memo += datum[:consumed_writes] - datum[:provisioned_writes]
+        end
+
+        memo
+      end
+    end
+
+    def total_read_units
+      @data.inject(0) do |memo, (_, datum)|
+        memo += datum[:provisioned_reads] if datum[:provisioned_reads]
+        memo
+      end
+    end
+
+    def total_write_units
+      @data.inject(0) do |memo, (_, datum)|
+        memo += datum[:provisioned_writes] if datum[:provisioned_writes]
+        memo
+      end
+    end
+
+    def wasted_read_percent
+      (wasted_read_units / total_read_units) * 100.0
+    end
+
+    def wasted_write_percent
+      (wasted_write_units / total_write_units) * 100.0
+    end
+
+    def lost_write_percent
+      (lost_write_units / total_write_units) * 100.0
+    end
+
+    def lost_read_percent
+      (lost_read_units / total_read_units) * 100.0
+    end
+
+    # Returns an array of all of the time points that have data present in
+    # them. Example:
+    #
+    #    table.tick(Time.now, { ... })
+    #    table.tick(Time.now, { ... })
+    #
+    #    table.all_times
+    #    #=> Array with the 2 time values above in it
+    def all_times
+      @data.keys
+    end
+
+    # Returns the earliest point in time that we have tracked data for.
+    def earliest_data_time
+      all_times.first
+    end
+
+    # Returns the latest point in time that we have tracked data for.
+    def latest_data_time
+      all_times.last
+    end
+
+    # Pricing is pretty difficult. This isn't a good measure of success. Base
+    # calculations on how many units are wasted.
+    # def wasted_money
+    #   UnitCost.read(wasted_read_units) + UnitCost.write(wasted_write_units)
+    # end
+
+    def to_csv! opts = {}
+      path = opts[:path] or File.join(DynamoAutoscale.root, "#{self.name}.csv")
+
+      CSV.open(path, 'w') do |csv|
+        csv << [
+          "time",
+          "provisioned_reads",
+          "provisioned_writes",
+          "consumed_reads",
+          "consumed_writes",
+        ]
+
+        @data.each do |time, datum|
+          csv << [
+            time.iso8601,
+            datum[:provisioned_reads],
+            datum[:provisioned_writes],
+            datum[:consumed_reads],
+            datum[:consumed_writes],
+          ]
+        end
+      end
+
+      path
+    end
+
+    def graph! opts = {}
+      data_tmp = File.join(Dir.tmpdir, 'data.csv')
+      png_tmp  = opts[:path] || File.join(Dir.tmpdir, 'graph.png')
+      r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_graph.r')
+
+      to_csv!(path: data_tmp)
+
+      `r --no-save --args #{data_tmp} #{png_tmp} < #{r_script}`
+
+      if $? != 0
+        logger.error "Failed to create graph."
+      else
+        `open #{png_tmp}` if opts[:open]
+      end
+
+      png_tmp
+    end
+
+    def scatterplot_for! metric
+      data_tmp = File.join(Dir.tmpdir, 'data.csv')
+      png_tmp  = File.join(Dir.tmpdir, 'boxplot.png')
+      r_script = File.join(DynamoAutoscale.root, 'rlib', 'dynamodb_boxplot.r')
+
+      to_csv!(data_tmp)
+
+      `r --no-save --args #{data_tmp} #{png_tmp} < #{r_script}`
+
+      if $? != 0
+        logger.error "Failed to create graph."
+      else
+        `open #{png_tmp}`
+      end
+    end
+
+    def report!
+      puts "         Table: #{name}"
+      puts "Wasted r/units: #{wasted_read_units.round(2)} (#{wasted_read_percent.round(2)}%)"
+      puts " Total r/units: #{total_read_units.round(2)}"
+      puts "  Lost r/units: #{lost_read_units.round(2)} (#{lost_read_percent.round(2)}%)"
+      puts "Wasted w/units: #{wasted_write_units.round(2)} (#{wasted_write_percent.round(2)}%)"
+      puts " Total w/units: #{total_write_units.round(2)}"
+      puts "  Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
+      puts "      Upscales: #{DynamoAutoscale.actioners[self].upscales}"
+      puts "    Downscales: #{DynamoAutoscale.actioners[self].downscales}"
+      puts "       Fitness: #{fitness}"
+    end
+
+    def fitness
+      lost_weight   = 100
+      wasted_weight = 1
+      lost          = ((lost_read_percent + lost_write_percent) / 2)
+      wasted        = ((wasted_read_percent + wasted_write_percent) / 2)
+
+      ((lost * lost_weight) + (wasted * wasted_weight)) / (lost_weight + wasted_weight)
+    end
+  end
+end

data/lib/dynamo-autoscale/unit_cost.rb

@@ -0,0 +1,41 @@
+module DynamoAutoscale
+  class UnitCost
+    # Pricing information obtained from: http://aws.amazon.com/dynamodb/pricing/
+    HOURLY_PRICING = {
+      'us-east-1' => {
+        read:  { dollars: 0.0065, per: 50 },
+        write: { dollars: 0.0065, per: 10 },
+      },
+      'us-west-1' => {
+        read:  { dollars: 0.0065, per: 50 },
+        write: { dollars: 0.0065, per: 10 },
+      },
+    }
+
+    # Returns the cost of N read units for an hour in a given region, which
+    # defaults to whatever is in
+    # DynamoAutoscale::DEFAULT_AWS_REGION.
+    #
+    # Example:
+    #
+    #   DynamoAutoscale::UnitCost.read(500, region: 'us-west-1')
+    #   #=> 0.065
+    def self.read units, opts = {}
+      pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:read]
+      ((units / pricing[:per].to_f) * pricing[:dollars])
+    end
+
+    # Returns the cost of N write units for an hour in a given region, which
+    # defaults to whatever is in
+    # DynamoAutoscale::DEFAULT_AWS_REGION.
+    #
+    # Example:
+    #
+    #   DynamoAutoscale::UnitCost.write(500, region: 'us-west-1')
+    #   #=> 0.325
+    def self.write units, opts = {}
+      pricing = HOURLY_PRICING[opts[:region] || DEFAULT_AWS_REGION][:write]
+      ((units / pricing[:per].to_f) * pricing[:dollars])
+    end
+  end
+end

data/lib/dynamo-autoscale/version.rb

@@ -0,0 +1,3 @@
+module DynamoAutoscale
+  VERSION = '0.1'
+end

data/lib/dynamo-autoscale.rb

@@ -0,0 +1 @@
+require_relative '../config/environment/common'

data/rlib/dynamodb_graph.r

@@ -0,0 +1,15 @@
+require(ggplot2)
+require(reshape)
+args <- commandArgs(trailingOnly = TRUE)
+data = read.csv(args[1], header=T, sep=",")
+
+data$time = strptime(data$time, "%Y-%m-%dT%H:%M:%SZ")
+
+measure.vars = c('provisioned_reads','provisioned_writes',
+                 'consumed_reads','consumed_writes')
+
+ive.melted = melt(data, id.vars='time', measure.vars = measure.vars)
+
+g = ggplot(ive.melted, aes(x=time, y=value, color=variable)) + geom_line()
+
+ggsave(file=args[2], plot=g, width=20, height=8)

data/rlib/dynamodb_scatterplot.r

@@ -0,0 +1,13 @@
+require(ggplot2)
+require(reshape)
+args <- commandArgs(trailingOnly = TRUE)
+data = read.csv(args[1], header=T, sep=",")
+
+data$time = strptime(data$time, "%Y-%m-%dT%H:%M:%SZ")
+data$hour = as.factor(strftime(data$time, "%H"))
+measure.vars = c(args[3])
+
+ive.melted = melt(data, id.vars='hour', measure.vars = measure.vars)
+g = ggplot(ive.melted, aes(x=hour, y=value, color=variable)) + geom_point()
+
+ggsave(file=args[2], plot=g, width=10, height=8)

data/rulesets/default.rb

@@ -0,0 +1,5 @@
+reads  last: 1, greater_than: "90%", scale: { on: :consumed, by: 2 }
+writes last: 1, greater_than: "90%", scale: { on: :consumed, by: 2 }
+
+reads  for:  2.hours, less_than: "50%", min: 2, scale: { on: :consumed, by: 2 }
+writes for:  2.hours, less_than: "50%", min: 2, scale: { on: :consumed, by: 2 }

data/rulesets/erroneous.rb

@@ -0,0 +1 @@
+reads last: 1, greater_than: 10, less_than: 5

data/rulesets/gradual_tail.rb

@@ -0,0 +1,11 @@
+reads  last: 2, greater_than: "90%", scale: { on: :consumed, by: 1.7 }
+reads  last: 2, greater_than: "80%", scale: { on: :consumed, by: 1.5 }
+
+writes last: 2, greater_than: "90%", scale: { on: :consumed, by: 1.7 }
+writes last: 2, greater_than: "80%", scale: { on: :consumed, by: 1.5 }
+
+reads  for:  2.hours, less_than: "20%", min: 10, scale: { on: :consumed, by: 1.8 }
+reads  for:  2.hours, less_than: "30%", min: 10, scale: { on: :consumed, by: 1.8 }
+
+writes for:  2.hours, less_than: "20%", min: 10, scale: { on: :consumed, by: 1.8 }
+writes for:  2.hours, less_than: "30%", min: 10, scale: { on: :consumed, by: 1.8 }

data/script/console

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+irb -r ./config/environment/console.rb

data/script/historic_data

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment/common'
+require 'pp'
+require 'fileutils'
+
+include DynamoAutoscale
+
+tables = ARGV
+
+if tables.empty?
+  STDERR.puts "Usage: script/historic_data table_name [another_table_name ...]"
+  exit 1
+end
+
+dynamo = AWS::DynamoDB.new
+range  = (Date.today - 5.days).upto(Date.today)
+logger.info "Date range: #{range.to_a}"
+
+# Filter out tables that do not exist in Dynamo.
+tables.select! do |table|
+  if dynamo.tables[table].exists?
+    true
+  else
+    logger.error "Table #{table} does not exist. Skipping."
+    false
+  end
+end
+
+range.each do |start_day|
+  dir     = File.join(DynamoAutoscale.data_dir, start_day.to_s)
+  end_day = start_day + 1.day
+
+  FileUtils.mkdir(dir) unless Dir.exists?(dir)
+
+  tables.each do |table|
+    logger.info "Collecting data for #{table} on #{start_day}..."
+    File.open(File.join(dir, "#{table}.json"), 'w') do |file|
+      file.write(JSON.pretty_generate(Metrics.all_metrics(table, {
+        period:     5.minutes,
+        start_time: start_day,
+        end_time:   end_day,
+      })))
+    end
+  end
+end

data/script/hourly_wastage

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+# This script calculate an approximate "wastage cost" for every table (wastage
+# cost is defined as provisioned throughout - consumed throughput, so throughput
+# that was paid for but not used).
+
+require_relative '../config/environment/common'
+include DynamoAutoscale
+
+tables      = AWS::DynamoDB.new.tables.to_a.map(&:name)
+pad         = tables.map(&:length).max
+total_waste = 0
+opts        = { period: 1.hour, start_time: 1.hour.ago, end_time: Time.now }
+
+tables.each do |table|
+  pr = Metrics.provisioned_reads(table, opts).map do |datum|
+    datum[:average]
+  end.inject(:+) || 0.0
+
+  pw = Metrics.provisioned_writes(table, opts).map do |datum|
+    datum[:average]
+  end.inject(:+) || 0.0
+
+  cr = Metrics.consumed_reads(table, opts).map do |datum|
+    datum[:average]
+  end.inject(:+) || 0.0
+
+  cw = Metrics.consumed_writes(table, opts).map do |datum|
+    datum[:average]
+  end.inject(:+) || 0.0
+
+  waste_cost   = UnitCost.read(pr - cr) + UnitCost.write(pw - cw)
+  total_waste += waste_cost
+
+  puts "#{table.rjust(pad)}: reads(#{cr.round(4)} / #{pr.round(4)}) " +
+       "writes(#{cw.round(4)} / #{pw.round(4)}), ~$#{waste_cost.round(4)} " +
+       "wasted per hour"
+end
+
+puts "Total waste cost: ~$#{total_waste.round(4)} per hour"

data/script/monitor

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+# This script is a sort-of REPL for monitoring production data. It will
+# periodically poll for production data and set off any alarms you define below.
+# Useful for testing autoscaling policies.
+#
+# At any point of the running process, pressing ctrl + c will pause execution
+# and drop you into a REPL where you can inspect the state of the program.
+# Exiting the REPL will resume execution.
+#
+# To exit the program entirely, drop into the REPL and run "exit!".
+
+require_relative '../config/environment/common'
+require 'pp'
+require 'timecop'
+include DynamoAutoscale
+
+ruleset = ARGV.shift
+tables  = ARGV
+
+if tables.empty? or ruleset.nil?
+  STDERR.puts "Usage: script/monitor ruleset table_name [another_table_name ... ]"
+  exit 1
+end
+
+# These filters use the arrays inside the local actioner to fake the provisioned
+# reads and writes when the local data enters the system. It makes it look like
+# we're actually modifying the provisioned numbers.
+filters = [
+  Proc.new do |time, datum|
+    if writes = DynamoAutoscale.actioner.provisioned_writes(table).last
+      datum[:provisioned_writes] = writes.last
+    end
+
+    if reads = DynamoAutoscale.actioner.provisioned_reads(table).last
+      datum[:provisioned_reads] = reads.last
+    end
+  end,
+]
+
+DynamoAutoscale.rules          = RuleSet.new(ruleset)
+DynamoAutoscale.dispatcher     = Dispatcher.new
+DynamoAutoscale.poller         = CWPoller.new(tables: tables)
+DynamoAutoscale.actioner_class = LocalActioner
+
+begin
+  DynamoAutoscale.poller.run
+rescue SignalException, Interrupt
+  Ripl.start :binding => binding
+  retry
+rescue => e
+  # If we error out, print the error and drop into a repl.
+  logger.error "Exception occurred: #{e.class}:#{e.message}"
+  Ripl.start :binding => binding
+end

data/script/simulator

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+# The simulator script reads from the data/ directory for historic data gathered
+# from CloudWatch with the script/historic_data script. Then it will step
+# through the data chronologically, dropping you into a REPL with each new
+# timeslice.
+#
+# Useful for checking that code works without having to make calls to the
+# CloudWatch API.
+
+require_relative '../config/environment/common'
+require 'timecop'
+require 'pp'
+include DynamoAutoscale
+
+ruleset = ARGV.shift
+tables  = ARGV
+
+if ruleset.nil? or tables.empty?
+  STDERR.puts "Usage: script/simulator ruleset table_name [another_table_name ... ]"
+  exit 1
+end
+
+filters                        = LocalActioner.faux_provisioning_filters
+DynamoAutoscale.rules          = RuleSet.new(ruleset)
+DynamoAutoscale.dispatcher     = Dispatcher.new
+DynamoAutoscale.poller         = LocalDataPoll.new(tables: tables, filters: filters)
+DynamoAutoscale.actioner_class = LocalActioner
+
+DynamoAutoscale.poller.run do |table, time, datum|
+  Timecop.travel(time)
+
+  puts "Event at #{time}"
+  puts "#{datum.pretty_inspect}"
+  puts
+  puts "Press ctrl + d or type 'exit' to step forward in time."
+  puts "Type 'exit!' to exit entirely."
+
+  Ripl.start :binding => binding
+end

data/script/test

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment/common'
+require 'pp'
+require 'timecop'
+include DynamoAutoscale
+extend  DynamoAutoscale
+
+graph   = !!(ARGV.delete("--graph") or ARGV.delete("-g"))
+ruleset = ARGV.shift
+tables  = ARGV
+
+if tables.empty? or ruleset.nil?
+  STDERR.puts "Usage: script/test ruleset table_name [another_table_name ... ] [-g|--graph]"
+  exit 1
+end
+
+filters                        = LocalActioner.faux_provisioning_filters
+DynamoAutoscale.rules          = RuleSet.new(ruleset)
+DynamoAutoscale.dispatcher     = Dispatcher.new
+DynamoAutoscale.poller         = LocalDataPoll.new(tables: tables, filters: filters)
+DynamoAutoscale.actioner_class = LocalActioner
+DynamoAutoscale.actioner_opts  = { group_downscales: true, flush_after: 2.hours }
+
+# Uncomment this and the below RubyProf lines if you want profiling information.
+# RubyProf.start
+
+begin
+  DynamoAutoscale.poller.run do |table_name, time|
+    Timecop.travel(time)
+  end
+rescue Interrupt
+  Ripl.start binding: binding
+end
+
+# Uncomment these and the above RubyProf line if you want profiling information.
+# printer = RubyProf::FlatPrinter.new(RubyProf.stop)
+# printer.print(STDOUT, min_percent: 2)
+
+# Uncomment this if you want to drop into a REPL at the end of the test.
+# Ripl.start binding: binding
+
+tables.each do |table_name|
+  table = DynamoAutoscale.tables[table_name]
+
+  table.report!
+
+  if graph
+    path = table.graph! open: true
+    puts "Graph saved to #{path}"
+  end
+end

data/script/validate_ruleset

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment/common'
+include DynamoAutoscale
+extend  DynamoAutoscale
+
+ruleset = ARGV.shift
+
+if ruleset.nil?
+  STDERR.puts "Usage: script/validate_ruleset ruleset"
+  exit 1
+end
+
+begin
+  RuleSet.new(ruleset)
+  puts "Rules seem legit."
+rescue ArgumentError => e
+  puts "Ruleset has problem: #{e.message}"
+  exit 1
+end