dynamo-autoscale 0.1.4 → 0.2

data/config/services/logger.rb

@@ -1,35 +1,31 @@
-DynamoAutoscale.with_config 'logger' do |config|
-  if config[:sync]
-    STDOUT.sync = true
-    STDERR.sync = true
-  end
+config = DynamoAutoscale.config[:logger] || {}
 
-  if config[:log_to]
-    STDOUT.reopen(config[:log_to])
-    STDERR.reopen(config[:log_to])
-  end
-
-  DynamoAutoscale::Logger.logger = ::Logger.new(STDOUT)
+if config[:sync]
+  STDOUT.sync = true
+  STDERR.sync = true
+end
 
-  if ENV['PRETTY_LOG']
-    DynamoAutoscale::Logger.logger.formatter = DynamoAutoscale::PrettyFormatter.new
-  else
-    DynamoAutoscale::Logger.logger.formatter = Logger::Formatter.new
-  end
+if config[:log_to]
+  STDOUT.reopen(config[:log_to])
+  STDERR.reopen(config[:log_to])
+end
 
-  if ENV['DEBUG']
-    DynamoAutoscale::Logger.logger.level = ::Logger::DEBUG
-  elsif config[:level]
-    DynamoAutoscale::Logger.logger.level = ::Logger.const_get(config[:level])
-  end
+DynamoAutoscale::Logger.logger = ::Logger.new(STDOUT)
 
-  if ENV['SILENT'] # or DynamoAutoscale.env == "test"
-    DynamoAutoscale::Logger.logger.level = ::Logger::FATAL
-  end
+if config[:style] == "pretty"
+  DynamoAutoscale::Logger.logger.formatter = DynamoAutoscale::PrettyFormatter.new
+else
+  DynamoAutoscale::Logger.logger.formatter = Logger::Formatter.new
 end
 
 if ENV['DEBUG']
-  AWS.config({
-    logger: DynamoAutoscale::Logger.logger,
-  })
+  DynamoAutoscale::Logger.logger.level = ::Logger::DEBUG
+elsif config[:level]
+  DynamoAutoscale::Logger.logger.level = ::Logger.const_get(config[:level])
 end
+
+if ENV['SILENT']
+  DynamoAutoscale::Logger.logger.level = ::Logger::FATAL
+end
+
+AWS.config(logger: DynamoAutoscale::Logger.logger) if ENV['DEBUG']

data/config/services/signals.rb

@@ -0,0 +1,15 @@
+Signal.trap("USR1") do
+  DynamoAutoscale.logger.info "[signal] Caught SIGUSR1. Dumping CSV for all tables in #{Dir.pwd}"
+
+  DynamoAutoscale.tables.each do |name, table|
+    table.to_csv! path: File.join(Dir.pwd, "#{table.name}.csv")
+  end
+end
+
+Signal.trap("USR2") do
+  DynamoAutoscale.logger.info "[signal] Caught SIGUSR2. Dumping graphs for all tables in #{Dir.pwd}"
+
+  DynamoAutoscale.tables.each do |name, table|
+    table.graph! path: File.join(Dir.pwd, "#{table.name}.png")
+  end
+end

data/dynamo-autoscale.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'ruby-prof'
   gem.add_dependency 'colored'
   gem.add_dependency 'activesupport'
+  gem.add_dependency 'pony'
 
   # ensure the gem is built out of versioned files
   gem.files = `git ls-files -z`.split("\0")

data/dynamo-autoscale.sample.yml

@@ -0,0 +1,79 @@
+:aws:
+  # REQUIRED
+  #
+  # Here is where you specify your AWS ID and key. It needs to be a pair that
+  # can access both CloudWatch and DynamoDB. For exact details on IAM policies,
+  # check the project README.
+  :access_key_id:      "your_id"
+  :secret_access_key:  "your_key"
+
+  # REQUIRED
+  #
+  # At the moment, only the us-east-1 region is supported. Changing this may
+  # have strange behvaviour. Don't do it.
+  :dynamo_db_endpoint: "dynamodb.us-east-1.amazonaws.com"
+
+:logger:
+  # Pretty logging makes the log output colourful. Who doesn't like colours?
+  :style: pretty
+  :level: INFO
+
+# If you want to receive email reports whenever a scale operation happens, you
+# can specify this email config. If you don't want to receive emails, just
+# remove this from your config file.
+#
+# dynamo-autoscale uses Pony to send email, this hash is just given to Pony
+# verbatim. Further documentation on what options Pony accepts can be found on
+# their GitHub: https://github.com/benprew/pony
+:email:
+  :to: "john.doe@example.com"
+  :from: "dynamo-autoscale@example.com"
+  :via: :smtp
+  :via_options:
+    :port: 25
+    :enable_starttls_auto: false
+    :authentication: :plain
+    :address: "mailserver.example.com"
+    :user_name: "user"
+    :password: "password"
+
+# REQUIRED
+#
+# Specify the path to your ruleset. Further information on the syntax and
+# purpose of rulesets can be found in the README.
+:ruleset: "rulesets/gradual_tail.rb"
+
+# REQUIRED
+#
+# The following is an array of tables to monitor and autoscale. You need to
+# specify at least one.
+:tables:
+  - "products-api-production4_dodgy_product_weights"
+
+# If you don't want to scale dynamo just yet, you can say that you want to run
+# in "dry run" mode, which will monitor your production databases but all of the
+# scaling events will only be tracked locally.
+#
+# Very useful for making sure your rules will do the right thing.
+:dry_run: true
+
+# Because you are very limited by how many downscales you have per day, and
+# because downscaling both reads and writes at the same time only counts as a
+# single downscale, the following option will queue up downscales until it can
+# apply 1 for reads and 1 for writes at the same time. It is recommended that
+# you turn this one.
+:group_downscales: true
+
+# This option only works in conjunction with the above group_downscales option.
+# If a downscale stays queued for a long time, you can specify a timeout and
+# just apply a single read or write downscale after a specified amount of time
+# passes.
+#
+# Specified in seconds.
+:flush_after: 3600
+
+# The following two options are configurable minimums and maximums for
+# provisioned throughputs. Dynamo-autoscale will not go below or above whatever
+# you set here.
+:minimum_throughput: 10
+:maximum_throughput: 20000

data/lib/dynamo-autoscale/actioner.rb

@@ -1,6 +1,7 @@
 module DynamoAutoscale
   class Actioner
     include DynamoAutoscale::Logger
+    attr_accessor :table, :upscales, :downscales
 
     def self.minimum_throughput
       @minimum_throughput ||= 10
@@ -18,8 +19,6 @@ module DynamoAutoscale
       @maximum_throughput = new_maximum_throughput
     end
 
-    attr_accessor :table, :upscales, :downscales
-
     def initialize table, opts = {}
       @table            = table
       @downscales       = 0
@@ -63,6 +62,13 @@ module DynamoAutoscale
       @last_scale_check = now
     end
 
+    # This should be overwritten by deriving classes. In the Dynamo actioner,
+    # this should check that the table is in an :active state. In the local
+    # actioner this will be faked.
+    def can_run?
+      false
+    end
+
     def upscales
       check_day_reset!
       @upscales
@@ -129,6 +135,7 @@ module DynamoAutoscale
       if result = scale(metric, to)
         @provisioned[metric][Time.now.utc] = to
         @upscales += 1
+        ScaleReport.new(table).send
       end
 
       return result
@@ -146,9 +153,8 @@ module DynamoAutoscale
       end
 
       if @pending[metric]
-        previous_pending = @pending[metric].last
         logger.info "[#{metric}][scaling down] " +
-          "#{previous_pending} -> #{to.round(2)} (overwritten pending)"
+          "#{@pending[metric]} -> #{to.round(2)} (overwritten pending)"
       else
         logger.info "[#{metric}][scaling down] " +
           "#{from ? from.round(2) : "Unknown"} -> #{to.round(2)}"
@@ -158,8 +164,7 @@ module DynamoAutoscale
     end
 
     def queue_operation! metric, value
-      @pending[metric] = [Time.now.utc, value]
-
+      @pending[metric] = value
       try_flush!
     end
 
@@ -168,6 +173,8 @@ module DynamoAutoscale
         if flush_operations!
           @downscales += 1
           @last_action = Time.now.utc
+          ScaleReport.new(table).send
+
           return true
         else
           return false
@@ -179,31 +186,38 @@ module DynamoAutoscale
 
     def flush_operations!
       result = nil
+      now = Time.now.utc
 
       if @pending[:writes] and @pending[:reads]
-        _, wvalue = @pending[:writes]
-        _, rvalue = @pending[:reads]
+        wvalue = @pending[:writes]
+        rvalue = @pending[:reads]
 
         if result = scale_both(rvalue, wvalue)
-          @provisioned[:writes][Time.now.utc] = wvalue
-          @provisioned[:reads][Time.now.utc] = rvalue
+          @provisioned[:writes][now] = wvalue
+          @provisioned[:reads][now] = rvalue
+
+          table.scale_events[now] = {
+            writes: wvalue,
+            reads: rvalue,
+          }
 
           @pending[:writes] = nil
           @pending[:reads] = nil
         end
       elsif @pending[:writes]
-        time, value = @pending[:writes]
+        value = @pending[:writes]
 
         if result = scale(:writes, value)
           @provisioned[:writes][Time.now.utc] = value
-
+          table.scale_events[now] = { writes: value }
           @pending[:writes] = nil
         end
       elsif @pending[:reads]
-        time, value = @pending[:reads]
+        value = @pending[:reads]
 
         if result = scale(:reads, value)
           @provisioned[:reads][Time.now.utc] = value
+          table.scale_events[now] = { reads: value }
           @pending[:reads] = nil
         end
       end

data/lib/dynamo-autoscale/cw_poller.rb

@@ -1,5 +1,6 @@
 module DynamoAutoscale
   class CWPoller < Poller
+    include DynamoAutoscale::Logger
     INTERVAL = 1.minute
 
     def poll tables, &block

data/lib/dynamo-autoscale/dispatcher.rb

@@ -1,5 +1,7 @@
 module DynamoAutoscale
   class Dispatcher
+    include DynamoAutoscale::Logger
+
     def initialize
       @last_check = {}
     end
@@ -26,8 +28,14 @@ module DynamoAutoscale
       block.call(table, time, datum) if block
 
       if @last_check[table.name].nil? or @last_check[table.name] < time
-        DynamoAutoscale.rules.test(table)
-        @last_check[table.name] = time
+        if DynamoAutoscale.actioners[table].can_run?
+          logger.debug "[dispatcher] Checking rules..."
+          DynamoAutoscale.rules.test(table)
+          @last_check[table.name] = time
+        else
+          logger.debug "[dispatcher] Skipped rule check, table is not ready " +
+            "to have its throughputs modified."
+        end
       else
         logger.debug "[dispatcher] Skipped rule check, already checked for " +
           "a later data point."

data/lib/dynamo-autoscale/dynamo_actioner.rb

@@ -1,5 +1,7 @@
 module DynamoAutoscale
   class DynamoActioner < Actioner
+    include DynamoAutoscale::Logger
+
     def dynamo
       @dynamo ||= AWS::DynamoDB.new.tables[table.name]
     end
@@ -19,14 +21,13 @@ module DynamoAutoscale
       dynamo_scale(read_capacity_units: reads, write_capacity_units: writes)
     end
 
+    def can_run?
+      dynamo.status == :active
+    end
+
     private
 
     def dynamo_scale opts
-      if dynamo.status == :updating
-        logger.warn "[actioner] Cannot scale throughputs. Table is updating."
-        return false
-      end
-
       dynamo.provision_throughput(opts)
       return true
     rescue AWS::DynamoDB::Errors::ValidationException => e

data/lib/dynamo-autoscale/local_actioner.rb

@@ -1,14 +1,28 @@
 module DynamoAutoscale
   class LocalActioner < Actioner
-    # Dummy scaling method.
+    include DynamoAutoscale::Logger
+
     def scale metric, value
+      @updating_until = rand(4.0..7.0).minutes.from_now.utc
       return true
     end
 
     def scale_both reads, writes
+      @updating_until = rand(4.0..7.0).minutes.from_now.utc
       return true
     end
 
+    def can_run?
+      return true if @updating_until.nil?
+
+      if Time.now.utc > @updating_until
+        @updating_until = nil
+        return true
+      end
+
+      return false
+    end
+
     # These filters use the arrays inside the local actioner to fake the
     # provisioned reads and writes when the local data enters the system. It
     # makes it look like we're actually modifying the provisioned numbers.
@@ -17,7 +31,6 @@ module DynamoAutoscale
         actioner = DynamoAutoscale.actioners[table]
 
         actioner.provisioned_reads.reverse_each do |rtime, reads|
-          logger.debug "Checking if #{time} > #{rtime}"
           if time > rtime
             logger.debug "[filter] Faked provisioned_reads to be #{reads} at #{time}"
             datum[:provisioned_reads] = reads

data/lib/dynamo-autoscale/local_data_poll.rb

@@ -1,5 +1,7 @@
 module DynamoAutoscale
   class LocalDataPoll < Poller
+    include DynamoAutoscale::Logger
+
     def initialize *args
       super(*args)
       @cache = Hash.new { |h, k| h[k] = {} }

data/lib/dynamo-autoscale/logger.rb

@@ -11,5 +11,9 @@ module DynamoAutoscale
     def logger
       DynamoAutoscale::Logger.logger
     end
+
+    def self.included base
+      base.extend DynamoAutoscale::Logger
+    end
   end
 end

data/lib/dynamo-autoscale/metrics.rb

@@ -1,6 +1,6 @@
 module DynamoAutoscale
   class Metrics
-    extend DynamoAutoscale::Logger
+    include DynamoAutoscale::Logger
 
     DEFAULT_OPTS = {
       namespace:   'AWS/DynamoDB',

data/lib/dynamo-autoscale/poller.rb

@@ -1,5 +1,8 @@
 module DynamoAutoscale
   class Poller
+    include DynamoAutoscale::Logger
+    attr_accessor :tables, :filters
+
     # The poller constructor accepts a hash of options. The following arguments
     # are valid but optional:
     #
@@ -9,11 +12,12 @@ module DynamoAutoscale
     #   each datum before it gets sent to the dispatcher. It helps fake setting
     #   provisioned throughput.
     def initialize opts = {}
-      @opts = opts
+      @tables  = opts[:tables] || []
+      @filters = opts[:filters] || []
     end
 
     def run &block
-      poll(@opts[:tables]) do |table_name, data|
+      poll(tables) do |table_name, data|
         logger.debug "[poller] Got data: #{data}"
         table = DynamoAutoscale.tables[table_name]
 
@@ -29,9 +33,7 @@ module DynamoAutoscale
             consumed_reads:     data[:consumed_reads][time],
           }
 
-          if @opts[:filters]
-            @opts[:filters].each { |filter| filter.call(table, time, datum) }
-          end
+          filters.each { |filter| filter.call(table, time, datum) }
 
           DynamoAutoscale.dispatcher.dispatch(table, time, datum, &block)
         end

data/lib/dynamo-autoscale/rule.rb

@@ -118,6 +118,7 @@ module DynamoAutoscale
             if @opts[:times].nil? or @count[table.name] == @opts[:times]
               @count[table.name] = 0
               logger.info "[rule] Triggered rule: #{self.to_english}"
+              table.triggered_rules[Time.now.utc] = self
 
               if scale = @opts[:scale]
                 new_val = table.send("last_#{scale[:on]}_for", @metric) * scale[:by]

data/lib/dynamo-autoscale/rule_set.rb

@@ -1,5 +1,7 @@
 module DynamoAutoscale
   class RuleSet
+    include DynamoAutoscale::Logger
+
     attr_accessor :rules
 
     def initialize path = nil, &block
@@ -23,10 +25,12 @@ module DynamoAutoscale
       rules  = self.for(table.name)
 
       rules.select(&:reads?).each do |rule|
+        # logger.debug "[rule_set] Checking rule: #{rule.to_english}"
         break result = true if rule.test(table)
       end
 
       rules.select(&:writes?).each do |rule|
+        # logger.debug "[rule_set] Checking rule: #{rule.to_english}"
         break result = true if rule.test(table)
       end
 

data/lib/dynamo-autoscale/scale_report.rb

@@ -0,0 +1,39 @@
+module DynamoAutoscale
+  class ScaleReport
+    include DynamoAutoscale::Logger
+
+    TEMPLATE = File.join(DynamoAutoscale.root, 'templates', 'scale_report_email.erb')
+
+    def initialize table
+      @table = table
+      @erb = ERB.new(File.read(TEMPLATE))
+
+      if config = DynamoAutoscale.config[:email]
+        @enabled = true
+        Pony.options = config
+      else
+        @enabled = false
+      end
+    end
+
+    def send
+      return false unless @enabled
+
+      result = Pony.mail({
+        subject: "Scale event for #{@table.name}",
+        body:    @erb.result(binding),
+      })
+
+      if result
+        logger.info "[mailer] Mail sent successfully."
+        result
+      else
+        logger.error "[mailer] Failed to send email. Result: #{result.inspect}"
+        false
+      end
+    rescue => e
+      logger.error "[mailer] Encountered an error: #{e.class}:#{e.message}"
+      false
+    end
+  end
+end

data/lib/dynamo-autoscale/table_tracker.rb

@@ -5,7 +5,7 @@ module DynamoAutoscale
     # TODO: This time window may need changing.
     TIME_WINDOW = 7.days
 
-    attr_reader :name, :data
+    attr_reader :name, :data, :triggered_rules, :scale_events
 
     def initialize name
       @name = name
@@ -13,7 +13,9 @@ module DynamoAutoscale
     end
 
     def clear_data
-      @data = RBTree.new
+      @data            = RBTree.new
+      @triggered_rules = RBTree.new
+      @scale_events    = RBTree.new
     end
 
     # `tick` takes two arguments. The first is a Time object, the second is
@@ -48,13 +50,9 @@ module DynamoAutoscale
       end
 
       @data[time] = datum
-
-      # The code below here just makes sure that we're trimming data points that
-      # are outside of the time window.
-      logger.debug "[table] Pruning data that may be outside of time window..."
-      now = Time.now.utc
-      to_delete = @data.each.take_while { |key, _| key < (now - TIME_WINDOW) }
-      to_delete.each { |key, _| @data.delete(key) }
+      remove_expired_data! @data
+      remove_expired_data! @triggered_rules
+      remove_expired_data! @scale_events
     end
 
     # Gets the last amount of provisioned throughput for whatever metric you
@@ -281,11 +279,18 @@ module DynamoAutoscale
 
       if $? != 0
         logger.error "[table] Failed to create graph."
+        return false
       else
-        `open #{png_tmp}` if opts[:open]
+        if opts[:open]
+          `open #{png_tmp}`
+          if $? != 0
+            logger.error "[table] Failed to open graph."
+            return false
+          else
+            return png_tmp
+          end
+        end
       end
-
-      png_tmp
     end
 
     def scatterplot_for! metric
@@ -314,16 +319,18 @@ module DynamoAutoscale
       puts "  Lost w/units: #{lost_write_units.round(2)} (#{lost_write_percent.round(2)}%)"
       puts "      Upscales: #{DynamoAutoscale.actioners[self].upscales}"
       puts "    Downscales: #{DynamoAutoscale.actioners[self].downscales}"
-      puts "       Fitness: #{fitness}"
     end
 
-    def fitness
-      lost_weight   = 100
-      wasted_weight = 1
-      lost          = ((lost_read_percent + lost_write_percent) / 2)
-      wasted        = ((wasted_read_percent + wasted_write_percent) / 2)
+    private
 
-      ((lost * lost_weight) + (wasted * wasted_weight)) / (lost_weight + wasted_weight)
+    # Helper function to remove data from an RBTree object keyed on a Time
+    # object where the key is outside of the time window defined by the
+    # TIME_WINDOW constant.
+    def remove_expired_data! data
+      # logger.debug "[table] Pruning data that may be outside of time window..."
+      now = Time.now.utc
+      to_delete = data.each.take_while { |key, _| key < (now - TIME_WINDOW) }
+      to_delete.each { |key, _| data.delete(key) }
     end
   end
 end

data/lib/dynamo-autoscale/version.rb

@@ -1,3 +1,3 @@
 module DynamoAutoscale
-  VERSION = '0.1.4'
+  VERSION = '0.2'
 end

data/script/historic_data

@@ -1,15 +1,24 @@
 #!/usr/bin/env ruby
 
-require_relative '../config/environment/common'
-require 'pp'
-require 'fileutils'
+# This script will fetch the 6 days of previous data from all of the tables that
+# you have specified in the config passed in as ARGV[0].
+#
+# It will store this data into the `data/` directory of this project in a format
+# that the rest of the tool scripts understands.
 
+require_relative '../config/environment/common'
 include DynamoAutoscale
 
-tables = ARGV
+if ARGV[0]
+  DynamoAutoscale.setup_from_config(ARGV[0])
+elsif ARGV[0].nil?
+  STDERR.puts "Usage: script/historic_data path/to/config.yml"
+
+  exit 1
+elsif ARGV[0] and !File.exists?(ARGV[0])
+  STDERR.puts "Usage: script/historic_data path/to/config.yml"
+  STDERR.puts "Error: The path you specified is to a file that does not exist."
 
-if tables.empty?
-  STDERR.puts "Usage: script/historic_data table_name [another_table_name ...]"
   exit 1
 end
 
@@ -18,7 +27,7 @@ range  = (Date.today - 5.days).upto(Date.today)
 logger.info "Date range: #{range.to_a}"
 
 # Filter out tables that do not exist in Dynamo.
-tables.select! do |table|
+DynamoAutoscale.poller.tables.select! do |table|
   if dynamo.tables[table].exists?
     true
   else
@@ -33,7 +42,7 @@ range.each do |start_day|
 
   FileUtils.mkdir(dir) unless Dir.exists?(dir)
 
-  tables.each do |table|
+  DynamoAutoscale.poller_opts[:tables].each do |table|
     logger.info "Collecting data for #{table} on #{start_day}..."
     File.open(File.join(dir, "#{table}.json"), 'w') do |file|
       file.write(JSON.pretty_generate(Metrics.all_metrics(table, {