dynamo-autoscale 0.3.6 → 0.4.1

data/script/random_test

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+# This script will locally test the tables and options you have specified in
+# your config passed in as ARGV[0].
+#
+# You will first need to have obtained historic data on the tables in your
+# config file. To do this, run:
+#
+#   $ script/historic_data path/to/config.yml
+#
+# This script does not change any throughputs on DynamoDB whatsoever. The
+# historic script data will hit CloudWatch fairly hard to get its data, though.
+
+require_relative '../config/environment/common'
+require 'timecop'
+
+OptionParser.new do |opts|
+  opts.on("--graph") do
+    GRAPH = true
+  end
+end.parse!
+
+if ARGV[0]
+  DynamoAutoscale.setup_from_config(ARGV[0], dry_run: true)
+elsif ARGV[0].nil?
+  STDERR.puts "Usage: script/test path/to/config.yml"
+
+  exit 1
+elsif ARGV[0] and !File.exists?(ARGV[0])
+  STDERR.puts "Usage: script/test path/to/config.yml"
+  STDERR.puts "Error: The path you specified is to a file that does not exist."
+
+  exit 1
+end
+
+# Uncomment this and the below RubyProf lines if you want profiling information.
+# RubyProf.start
+
+DynamoAutoscale.poller_class = DynamoAutoscale::RandomDataGenerator
+DynamoAutoscale.poller_opts  = {
+  num_points: 100,
+  start_time: Time.now,
+  provisioned_reads: 600,
+  provisioned_writes: 600,
+}.merge(DynamoAutoscale.poller_opts)
+
+begin
+  DynamoAutoscale.poller.run { |table_name, time| Timecop.travel(time) }
+rescue Interrupt
+  Ripl.start binding: binding
+end
+
+# Uncomment these and the above RubyProf line if you want profiling information.
+# printer = RubyProf::FlatPrinter.new(RubyProf.stop)
+# printer.print(STDOUT, min_percent: 2)
+
+# Uncomment this if you want to drop into a REPL at the end of the test.
+# Ripl.start binding: binding
+
+DynamoAutoscale.tables.each do |_, table|
+  table.report! metric: :cost
+
+  if Kernel.const_defined?(:GRAPH) and GRAPH
+    if path = table.graph!(open: true)
+      puts "Graph saved to #{path}"
+    end
+  end
+end

data/script/test

@@ -52,8 +52,9 @@ end
 DynamoAutoscale.tables.each do |_, table|
   table.report!
 
-  if GRAPH
-    path = table.graph! open: true
-    puts "Graph saved to #{path}"
+  if Kernel.const_defined?(:GRAPH) and GRAPH
+    if path = table.graph!(open: true)
+      puts "Graph saved to #{path}"
+    end
   end
 end

data/spec/config_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe 'configuration' do
+  it "should crash with no AWS region specified" do
+    expect do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH, aws: { })
+    end.to raise_error DynamoAutoscale::Error::InvalidConfigurationError
+  end
+
+  it "should warn when using a non standard AWS region" do
+    log = catch_logs do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH, {
+        logger: nil,
+        aws:    { region: "wut" }
+      })
+    end
+
+    bool = log[:warn].any? { |m| m.include?("wut") and m.include?("region") }
+    bool.should be_true
+  end
+
+  it "should set log level to debug when ENV['DEBUG'] is set" do
+    with_env("DEBUG" => "true") do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH)
+      DynamoAutoscale.logger.level.should == ::Logger::DEBUG
+    end
+  end
+
+  it "should set ENV['DEBUG'] log level even when no logger config is present" do
+    with_env("DEBUG" => "true") do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH, logger: nil)
+      DynamoAutoscale.logger.level.should == ::Logger::DEBUG
+    end
+  end
+
+  it "should set log level to fatal when ENV['SILENT'] is set" do
+    with_env("SILENT" => "true") do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH)
+      DynamoAutoscale.logger.level.should == ::Logger::FATAL
+    end
+  end
+
+  it "should set log level to info by default" do
+    with_env({}) do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH)
+      DynamoAutoscale.logger.level.should == ::Logger::INFO
+    end
+  end
+
+  it "DEBUG should take precedence over SILENT" do
+    with_env("SILENT" => "true", "DEBUG" => "true") do
+      DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH)
+      DynamoAutoscale.logger.level.should == ::Logger::DEBUG
+    end
+  end
+
+  it "should use a standard formatter when pretty is not specified" do
+    DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH, {
+      logger: {}
+    })
+
+    DynamoAutoscale.logger.formatter.should be_a DynamoAutoscale::StandardFormatter
+  end
+
+  it "should use a pretty formatter when pretty is specified" do
+    DynamoAutoscale.setup_from_config(TEST_CONFIG_PATH, {
+      logger: { style: "pretty" }
+    })
+
+    DynamoAutoscale.logger.formatter.should be_a DynamoAutoscale::PrettyFormatter
+  end
+end

data/spec/dispatcher_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe DynamoAutoscale::Dispatcher do
+  let(:table_name)   { "example_table" }
+  let(:table)        { DynamoAutoscale.tables[table_name] }
+  let(:poller_class) { DynamoAutoscale::FakePoller }
+  let(:poller_opts)  { { data: poller_data, tables: [table_name] } }
+
+  before do
+    DynamoAutoscale.poller_class = poller_class
+    DynamoAutoscale.poller_opts  = poller_opts
+  end
+
+  describe "simple dispatching" do
+    let(:time1) { Time.now }
+    let(:time2) { time1 + 15.minutes }
+
+    let :poller_data do
+      {
+        consumed_reads: {
+          time1 => 10,
+          time2 => 20,
+        }
+      }
+    end
+
+    it "should correctly dispatch data" do
+      DynamoAutoscale.poller.run
+
+      table.data[time1][:consumed_reads].should == 10
+      table.data[time2][:consumed_reads].should == 20
+    end
+  end
+
+  describe "dispatching out of order data" do
+    let(:time1) { Time.now }
+    let(:time2) { time1 + 15.minutes }
+
+    let :poller_data do
+      {
+        consumed_reads: {
+          time2 => 20,
+          time1 => 10,
+        }
+      }
+    end
+
+    it "should correctly dispatch data" do
+      DynamoAutoscale.poller.run
+
+      table.data[time1][:consumed_reads].should == 10
+      table.data[time2][:consumed_reads].should == 20
+      table.data.keys.should == [time1, time2]
+    end
+  end
+end

data/spec/helpers/environment_helper.rb

@@ -0,0 +1,15 @@
+module DynamoAutoscale
+  module Helpers
+    module EnvironmentHelper
+      def with_env env, &block
+        old_env = ENV.respond_to?(:to_h) ? ENV.to_h.dup : ENV.to_hash.dup
+        ENV.clear
+        env.each { |h, k| ENV[h] = k }
+        block.call
+        ENV.clear
+        old_env.each { |h, k| ENV[h] = k }
+        nil
+      end
+    end
+  end
+end

data/spec/helpers/logger.rb

@@ -0,0 +1,20 @@
+module DynamoAutoscale
+  module Helpers
+    module LoggerHelper
+      def catch_logs &block
+        old_logger             = DynamoAutoscale.logger
+        old_formatter          = old_logger.formatter
+        old_level              = old_logger.level
+        collector              = LogCollector.new(STDOUT)
+        collector.formatter    = old_formatter
+        collector.level        = old_level
+        DynamoAutoscale.logger = collector
+
+        block.call
+
+        DynamoAutoscale.logger = old_logger
+        collector.messages
+      end
+    end
+  end
+end

data/spec/poller_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe DynamoAutoscale::Poller do
+  let(:table_name)   { "example_table" }
+  let(:table)        { DynamoAutoscale.tables[table_name] }
+  let(:poller_class) { DynamoAutoscale::FakePoller }
+  let(:poller_opts)  { { data: poller_data, tables: [table_name] } }
+
+  before do
+    DynamoAutoscale.poller_class = poller_class
+    DynamoAutoscale.poller_opts  = poller_opts
+  end
+
+  describe "simple dispatching" do
+    let(:time1) { Time.now }
+    let(:time2) { time1 + 15.minutes }
+
+    let :poller_data do
+      {
+        consumed_reads: {
+          time1 => 10,
+          time2 => 20,
+        }
+      }
+    end
+
+    it "should correctly dispatch data" do
+      DynamoAutoscale.dispatcher.should_receive(:dispatch).with(
+        table, time1, { consumed_reads: 10 }
+      ).once
+
+      DynamoAutoscale.dispatcher.should_receive(:dispatch).with(
+        table, time2, { consumed_reads: 20 }
+      ).once
+
+      DynamoAutoscale.poller.run
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,4 +1 @@
 require_relative '../config/environment/test'
-
-RSpec.configure do |config|
-end

data/spec/table_tracker_spec.rb

@@ -51,6 +51,115 @@ describe DynamoAutoscale::TableTracker do
       it      { should == table_name }
     end
 
+    describe "#earliest_data_time" do
+      subject { table.earliest_data_time }
+      it      { should == table.data.keys.first }
+    end
+
+    describe "#total_read_units" do
+      subject { table.total_read_units }
+      it      { should == 1900 }
+    end
+
+    describe "#total_write_units" do
+      subject { table.total_write_units }
+      it      { should == 2600 }
+    end
+
+    describe "#lost_read_units" do
+      subject { table.lost_read_units }
+      it      { should == 0 }
+    end
+
+    describe "#lost_write_units" do
+      subject { table.lost_write_units }
+      it      { should == 0 }
+    end
+
+    describe "#wasted_read_units" do
+      subject { table.wasted_read_units }
+      it      { should == 1820 }
+    end
+
+    describe "#wasted_write_units" do
+      subject { table.wasted_write_units }
+      it      { should == 2480 }
+    end
+
+    context 'AWS region us-east' do
+      before { AWS.config(region: 'us-east-1') }
+
+      describe "#total_read_cost" do
+        subject { table.total_read_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#total_write_cost" do
+        subject { table.total_write_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#total_read_cost" do
+        subject { table.total_read_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#lost_write_cost" do
+        subject { table.lost_write_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#lost_read_cost" do
+        subject { table.lost_read_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#wasted_read_cost" do
+        subject { table.wasted_read_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+
+      describe "#wasted_write_cost" do
+        subject { table.wasted_write_cost }
+        it      { should be_a Float }
+        it      { should be >= 0 }
+      end
+    end
+
+    describe "#lost_write_percent" do
+      subject { table.lost_write_percent }
+      it      { should be_a Float }
+      it      { should be >= 0 }
+      it      { should be <= 100 }
+    end
+
+    describe "#lost_read_percent" do
+      subject { table.lost_read_percent }
+      it      { should be_a Float }
+      it      { should be >= 0 }
+      it      { should be <= 100 }
+    end
+
+    describe "#wasted_read_percent" do
+      subject { table.wasted_read_percent }
+      it      { should be_a Float }
+      it      { should be >= 0 }
+      it      { should be <= 100 }
+    end
+
+    describe "#wasted_write_percent" do
+      subject { table.wasted_write_percent }
+      it      { should be_a Float }
+      it      { should be >= 0 }
+      it      { should be <= 100 }
+    end
+
     describe "#last 3.seconds, :consumed_reads" do
       subject { table.last 3.seconds, :consumed_reads }
       it      { should == [20.0] }
@@ -76,12 +185,66 @@ describe DynamoAutoscale::TableTracker do
       it      { should == 800.0 }
     end
 
+    describe "#last_provisioned_for :writes, at: 3.hours.ago" do
+      subject { table.last_provisioned_for :writes, at: 3.hours.ago }
+      it      { should == nil }
+    end
+
+    describe "#last_consumed_for :reads" do
+      subject { table.last_consumed_for :reads }
+      it      { should == 20.0 }
+    end
+
+    describe "#last_consumed_for :writes, at: now" do
+      subject { table.last_consumed_for :writes, at: now }
+      it      { should == 30.0 }
+    end
+
+    describe "#last_consumed_for :writes, at: 3.minutes.ago" do
+      subject { table.last_consumed_for :writes, at: 3.minutes.ago }
+      it      { should == 30.0 }
+    end
+
+    describe "#last_consumed_for :writes, at: 3.hours.ago" do
+      subject { table.last_consumed_for :writes, at: 3.hours.ago }
+      it      { should == nil }
+    end
+
     describe "#all_times" do
       subject      { table.all_times }
       its(:length) { should == 4 }
 
       specify("is ordered") { subject.should == subject.sort }
     end
+
+    describe "#to_csv!" do
+      let(:tempfile) { Tempfile.new(table_name) }
+      subject        { File.readlines(table.to_csv!(path: tempfile.path)) }
+      its(:count)    { should == 5 }
+      after          { tempfile.unlink }
+    end
+
+    describe "#report!" do
+      context "metric: :units:" do
+        it "should not error" do
+          table.report! metric: :units
+        end
+      end
+
+      context "metric: :cost" do
+        it "should not error" do
+          table.report! metric: :cost
+        end
+      end
+
+      context "metric: :bananas" do
+        it "should error" do
+          expect do
+            table.report! metric: :bananas
+          end.to raise_error ArgumentError
+        end
+      end
+    end
   end
 
   describe 'clearing data' do