dynamo-autoscale 0.1

data/.gitignore

@@ -0,0 +1,4 @@
+config/aws.yml
+aws.yml
+data/
+pkg/

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'ripl'
+  gem 'timecop'
+end
+
+group :test do
+  gem 'rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,58 @@
+PATH
+  remote: .
+  specs:
+    dynamo-autoscale (0.1)
+      aws-sdk
+      colored
+      rbtree
+      ruby-prof
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
+      multi_json (~> 1.0)
+    aws-sdk (1.11.2)
+      json (~> 1.4)
+      nokogiri (< 1.6.0)
+      uuidtools (~> 2.1)
+    bond (0.4.3)
+    coderay (1.0.9)
+    colored (1.2)
+    diff-lcs (1.2.4)
+    i18n (0.6.1)
+    json (1.8.0)
+    method_source (0.8.1)
+    multi_json (1.7.6)
+    nokogiri (1.5.10)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rbtree (0.4.1)
+    ripl (0.7.0)
+      bond (~> 0.4.2)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    ruby-prof (0.13.0)
+    slop (3.4.5)
+    timecop (0.6.1)
+    uuidtools (2.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  dynamo-autoscale!
+  pry
+  ripl
+  rspec
+  timecop

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 InvisibleHand Software Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,400 @@
+# DynamoDB Autoscaling
+
+**IMPORTANT**: It's highly recommended that you read this README before
+continuing. This project, if used incorrectly, has a lot of potential to cost
+you huge amounts of money. Proceeding with caution is paramount, as we cannot be
+held responsible for misuse that leads to excessive cost on your part.
+
+There are tools and flags in place that will allow you to dry-run the project
+before actually allowing it to change your provisioned throughputs and it is
+highly recommended that you first try running the project as a dry-run and
+inspecting the log output to make sure it is doing what you expect.
+
+It is also worth noting that this project is very much in its infancy.
+
+You have been warned.
+
+## Rules of the game
+
+Welcome to the delightful mini-game that is DynamoDB provisioned throughputs.
+Here are the rules of the game:
+
+  - In a single API call, you can only change your throughput by up to 100% in
+  	either direction. In other words, you can decrease as much as you want but
+  	you can only increase to up to double what the current throughput is.
+
+  - You may scale up as many times per day as you like, however you may only
+  	scale down 4 times per day per table. (If you scale both reads and writes
+  	down in the same request, that only counts as 1 downscale used)
+
+  - Scaling is not an instantaneous event. It can take up to 5 minutes for a
+  	table's throughput to be updated.
+
+  - Small spikes over your threshold are tolerated but the exact amount of time
+  	they are tolerated for seems to vary.
+
+This project aims to take all of this into consideration and automatically scale
+your throughputs to enable you to deal with spikes and save money where
+possible.
+
+# Configuration
+
+This library requires AWS keys that have access to both CloudWatch and DynamoDB,
+for retriving data and sending scaling requests. Using IAM, create a new user, and
+assign the 'CloudWatch Read Only Access' policy template. In addition, you will
+need to use the Policy Generator to add at least the following Amazon DynamoDB actions:
+
+  - "dynamodb:DescribeTable"
+  - "dynamodb:ListTables"
+  - "dynamodb:UpdateTable"
+
+The ARN for the custom policy can be specified as '\*' to allow access to all tables,
+or alternatively you can refer to the IAM documentation to limit access to specific
+tables only.
+
+The project will look for a YAML file in the following locations on start up:
+
+  - ./aws.yml
+  - ENV['AWS_CONFIG']
+
+If it doesn't find an AWS YAML config in any of those locations, the process
+prints an error and exits.
+
+**A sample config can be found in the project root directory.**
+
+# Usage
+
+First of all, you'll need to install this project as a gem:
+
+    $ gem install dynamo-autoscale
+
+This will give you access to the `dynamo-autoscale` executable. For some
+internal documentation on the executable, you can run:
+
+    $ dynamo-autoscale -h
+
+This should tell you what flags you can set and what arguments the command
+expects.
+
+## Logging
+
+By default, not a whole lot will be logged at first. If you want to be sure that
+the gem is working and doing things, you can run with the `DEBUG` environment
+variable set to `true`:
+
+    $ DEBUG=true dynamo-autoscale <args...>
+
+Also, if you want pretty coloured logging, you can set the `PRETTY_LOG`
+environment variable to `true`:
+
+    $ PRETTY_LOG=true DEBUG=true dynamo-autoscale <args...>
+
+## Rulesets
+
+One of the first things you'll notice upon looking into the `--help` on the
+executable is that it's looking for a "rule set". What on earth is a rule set?
+
+A rule set is the primary user input for dynamo-autoscale. It is a DSL for
+specifying when to increase and decrease your provisioned throughputs. Here is a
+very basic rule set:
+
+``` ruby
+reads  last: 1, greater_than: "90%", scale: { on: :consumed, by: 2 }
+writes last: 1, greater_than: "90%", scale: { on: :consumed, by: 2 }
+
+reads  for:  2.hours, less_than: "50%", min: 2, scale: { on: :consumed, by: 2 }
+writes for:  2.hours, less_than: "50%", min: 2, scale: { on: :consumed, by: 2 }
+```
+
+You would put this ruleset in a file and then pass that file in as the first
+argument to `dynamo-autoscale` on the command line.
+
+The first two rules are designed to deal with spikes. They are saying that if
+the consumed capacity units is greater than 90% of the provisioned throughput
+for a single data point, scale the provisioned throughput up by the last
+consumed units multiplied by two.
+
+For example, if we had a provisioned reads of 100 and a consumed units of
+95 comes through, that will trigger that rule and the table will be scaled up to
+have a provisioned reads of 190.
+
+The last two rules are controlling downscaling. Because downscaling can only
+happen 4 times per day per table, the rules are far less aggressive. Those rules
+are saying: if the consumed capacity is less than 50% of the provisioned for a
+whole two hours, with a minimum of 2 data points, scale the provisioned
+throughput to the consumed units multiplied by 2.
+
+### The :last and :for options
+
+These options declare how many points or what time range you want to examine.
+They're aliases of each other and if you specify both, one will be ignored. If
+you don't specify a `:min` or `:max` option, they will just get as many points
+as they can and evaluate the rest of the rule even if they don't get a full 2
+hours of data, or a full 6 points of data. This only affects the start of the
+process's lifetime, eventually it will have enough data to always get the full
+range of points you're asking for.
+
+### The :min and :max options
+
+If you're not keen on asking for 2 hours of data and not receiving the full
+range before evaluating the rest of the rule, you can specify a minimum or
+maximum number of points to evaluate. Currently, this only supports a numeric
+value. So you can ask for at least 20 points to be present like so:
+
+``` ruby
+reads for: 2.hours, less_than: "50%", min: 20, scale: { on: :consumed, by: 2 }
+```
+
+### The :greater_than and :less_than options
+
+You must specify at least one of these options for the rule to actually validate
+without throwing an error. Having neither makes no sense.
+
+You can specify either an absolute value or a percentage specified as a string.
+The percentage will calculate the percentage consumed against the amount
+provisioned.
+
+Examples:
+
+``` ruby
+reads for: 2.hours, less_than: 10, scale: { on: :consumed, by: 2 }
+
+reads for: 2, less_than: "20%", scale: { on: :consumed, by: 2 }
+```
+
+### The :scale option
+
+The `:scale` option is a way of doing a simple change to the provisioned
+throughput without having to specify repetitive stuff in a block. `:scale`
+expects to be a hash and it expects to have two keys in the hash: `:on` and
+`:by`.
+
+`:on` specifies what part of the metric you want to scale on. It can either by
+`:provisioned` or `:consumed`. In most cases, `:consumed` makes a lot more sense
+than `:provisioned`.
+
+`:by` specifies the scale factor. If you want to double the provisioned capacity
+when a rule triggers, you would write something like this:
+
+``` ruby
+reads for: 2.hours, less_than: "30%", scale: { on: :provisioned, by: 0.5 }
+```
+
+And that would half the provisioned throughput for reads if the consumed is
+less than 30% of the provisioned for 2 hours.
+
+### Passing a block
+
+If you want to do something a little bit more complicated with your rules, you
+can pass a block to them. The block will get passed three things: the table the
+rule was triggered for, the rule object that triggered and the actioner for that
+table.
+
+An actioner is an abstraction of communication with Dynamo and it allows
+communication to be faked if you want to do a dry run. It exposes a very simple
+interface. Here's an example:
+
+``` ruby
+writes for: 2.hours, greater_than: 200 do |table, rule, actioner|
+  actioner.set(:writes, 300)
+end
+```
+
+This rule will set the provisioned write throughput to 300 if the consumed
+writes are greater than 200 for 2 hours. The actioner handles a tonne of things
+under the hood, such as making sure you don't scale up more than you're allowed
+to in a single call and making sure you don't try to change a table when it's in
+the updating state.
+
+It also handles the grouping of downscales, which we will talk about in a later
+section of the README.
+
+The `table` argument is a `TableTracker` object. For a run down of what
+information is available to you I advise checking out the source code in
+`lib/dynamo-autoscale/table_tracker.rb`.
+
+### The :count option
+
+The `:count` option allows you to specify that a rule must be triggered a set
+number of times in a row before its action is executed.
+
+Example:
+
+``` ruby
+writes for: 10.minutes, greater_than: "90%", count: 3, scale: { on: :consumed, by: 1.5 }
+```
+
+This says that is writes are greater than 90% for 10 minutes three checks in a
+row, scale by the amount consumed multiplied by 1.5. A new check will only
+happen when the table receives new data from cloud watch, which means that the
+10 minute windows could potentially overlap.
+
+## Downscale grouping
+
+You can downscale reads or writes individually and this will cost you one of
+your four downscales for the current day. Or, you can downscale reads and writes
+at the same time and this also costs you one of your four. (Reference:
+http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html)
+
+Because of this, the actioner can handle the grouping up of downscales. Let's
+say you passed in the following options in at the command line:
+
+    $ dynamo-autoscale some/ruleset.rb some_table --group-downscales --flush-after 300
+
+What this is saying is that if a write downscale came in, the actioner wouldn't
+fire it off immediately. It would wait 300 seconds, or 5 minutes, to see if a
+corresponding read downscale was triggered and would run them both at the same
+time. If no corresponding read came in, after 5 minutes the pending write
+downscale would get "flushed" and applied without a read downscale.
+
+This technique helps to save downscales on tables that may have unpredictable
+consumption. You may need to tweak the `--flush-after` value to match your own
+situation. By default, there is no `--flush-after` and downscales will wait
+indefinitely, this may not be desirable.
+
+## Signaling
+
+The `dynamo-autoscale` process responds to the SIGUSR1 and SIGUSR2 signals. What
+we've done may be a dramatic bastardisation of what signals are intended for or
+how they work, but here's what each does.
+
+### USR1
+
+If you send SIGUSR1 to the process as it's running, the process will dump all of
+the data it has collected on all of the tables it is collecting for into CSV
+files in the directory it was run in.
+
+Example:
+
+    $ dynamo-autoscale some/ruleset.rb some_table
+    # Runs as PID 1234. Wait for some time to pass...
+    $ kill -USR1 1234
+    $ cat some_table.csv
+
+The CSV is in the following format:
+
+    time,provisioned_reads,provisioned_writes,consumed_reads,consumed_writes
+    2013-07-02T10:48:00Z,800.0,600.0,390.93666666666667,30.54
+    2013-07-02T10:49:00Z,800.0,600.0,390.93666666666667,30.54
+    2013-07-02T10:53:00Z,800.0,600.0,386.4533333333333,95.26666666666667
+    2013-07-02T10:54:00Z,800.0,600.0,386.4533333333333,95.26666666666667
+    2013-07-02T10:58:00Z,800.0,600.0,110.275,25.406666666666666
+    2013-07-02T10:59:00Z,800.0,600.0,246.12,54.92
+
+### USR2
+
+If you send SIGUSR2 to the process as it's running, the process will take all of
+the data it has on all of its tables and generate a graph for each table using R
+(see the Graphs section below). This is handy for visualising what the process
+is doing, especially after doing a few hours of a `--dry-run`.
+
+# Developers / Tooling
+
+Everything below this part of the README is intended for people that want to
+work on the dynamo-autoscale codebase or use the internal tools that we use for
+testing new rulesets.
+
+## Technical details
+
+The code has a set number of moving parts that are globally available and must
+implement certain interfaces (for exact details, you would need to study the
+code):
+
+  - `DynamoAutoscale.poller`: This component is responsible for pulling data
+  	from a data source (CloudWatch or Local at the moment) and piping it into
+  	the next stage in the pipeline.
+
+  - `DynamoAutoscale.dispatcher`: The dispatcher takes data from the poller and
+  	populates a hash table of `TableTracker` objects, as well as checking to see
+  	if any of the tables have triggered any rules.
+
+  - `DynamoAutoscale.rules`: The ruleset contains an array of `Rule` objects
+  	inside a hash table keyed by table name. The ruleset initializer takes a
+  	file path as an argument, or a block, either of these needs to contain a set
+  	of rules (examples can be found in the `rulesets/` directory).
+
+  - `DynamoAutoscale.actioners`: The actioners are what perform provision scaling.
+  	Locally this is faked, in production it makes API calls to DynamoDB.
+
+  - `DynamoAutoscale.tables`: This is a hash table of `TableTracker` objects,
+  	keyed on the table name.
+
+All of these components are globally available because most of them need access
+to each other and it was a pain to pass instances of them around to everybody
+that needed them.
+
+They're also completely swappable. As long as they implement the right methods
+you can get your data from anywhere, dispatch your data to anywhere and send
+your actions to whatever you want. The defaults all work on local data gathered
+with the `script/historic_data` executable.
+
+## Testing rules locally
+
+If you want to test rules on your local machine without having to query
+CloudWatch or hit DynamoDB, there are tools that facilitate that nicely.
+
+The first thing you would need to do is gather some historic data. There's a
+script called `script/historic_data` that you can run to gather data on a
+specific table and store it into the `data/` directory in a format that all of
+the other scripts are familiar with.
+
+Next there are a couple of things you can do.
+
+### Running a test
+
+You can run a big batch of data all in one go with the `script/test` script.
+This script can be invoked like this:
+
+    $ script/test rulesets/default.rb table_name
+
+Substituting `table_name` with the name of a table that exists in your DynamoDB.
+This will run through all of the data for that table in time order, logging
+along the way and triggering rules from the rule set if any were defined.
+
+At the end, it shows you a report on the amount of wasted, used and lost units.
+
+#### Graphs
+
+If you felt so inclined, you could add the `--graph` flag to the above command
+and the script will generate a graph for you at the end. This will shell out to
+an R process to generate the graph, so you will need to ensure that you have R
+installed on your system with the `ggplot2` and `reshape` packages installed.
+
+Personally, I use a Mac and I attempted to install R through Homebrew but had
+troubles with compiling packages. I had far more success when I installed R
+straight from the R website, http://cran.r-project.org/bin/macosx/, and used
+their GUI R.app to install the packages.
+
+None of this is required to run the `dynamo-autoscale` executable in production.
+
+### Simulating data coming in
+
+There's a script called `script/simulator` that allows you to step through data
+as it arrives. It takes the exact same arguments as the `script/test` script but
+instead of running all the way through the data and generating a report,
+`script/simulate` will pause after each round of new data and drop you into a
+REPL. This is very handy for debugging tricky situations with your rules or the
+codebase.
+
+The simulator does not hit CloudWatch or DynamoDB at any point.
+
+## Contributing
+
+Report Issues/Feature requests on
+[GitHub Issues](https://github.com/invisiblehand/dynamo-autoscale/issues).
+
+#### Note on Patches/Pull Requests
+
+ * Fork the project.
+ * Make your feature addition or bug fix.
+ * Add tests for it. This is important so we don't break it in a
+   future version unintentionally.
+ * Commit, do not modify the rakefile, version, or history.
+   (if you want to have your own version, that is fine but bump version in a commit by itself so it can be ignored when we pull)
+ * Send a pull request. Bonus points for topic branches.
+
+### Copyright
+
+Copyright (c) 2013 InvisibleHand Software Ltd. See
+[LICENSE](https://github.com/invisiblehand/dynamo-autoscale/blob/master/LICENSE)
+for details.

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+
+task :default => [:test]
+
+desc "Run all tests"
+RSpec::Core::RakeTask.new(:test) do |t|
+  t.rspec_opts = '-cfs'
+end

data/aws.sample.yml

@@ -0,0 +1,16 @@
+default: &default
+  :access_key_id: your_id
+  :secret_access_key: your_key
+
+development:
+  <<: *default
+  :dynamo_db_endpoint: dynamodb.us-east-1.amazonaws.com
+
+test:
+  <<: *default
+  :dynamo_db_endpoint: localhost
+  :dynamo_db_port: 4568
+
+production:
+  <<: *default
+  :dynamo_db_endpoint: dynamodb.us-east-1.amazonaws.com

data/bin/dynamo-autoscale

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+
+require 'pp'
+require 'optparse'
+require 'active_support/all'
+
+# Force this script into production mode as it's the only thing that will
+# actually hit DynamoDB in the entire project.
+ENV['RACK_ENV'] = "production"
+
+actioner_opts = {}
+general_opts  = {}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: dynamo-autoscale ruleset_path table_name [more table names] [options]"
+
+  doc = 'Makes read and write downscales happen at the same time to save ' +
+    'downscales per day.'
+
+  opts.on('-g', '--group-downscales', doc) do
+    actioner_opts[:group_downscales] = true
+  end
+
+  doc = 'Only works in conjunction with --group-downscales. Sets a maximum ' +
+    'amount of time for an operation to be pending before it gets applied to Dynamo'
+
+  opts.on('--flush-after SECONDS', Integer, doc) do |seconds|
+    actioner_opts[:flush_after] = seconds.to_i.seconds
+  end
+
+  doc = 'Stops dynamo-autoscale from talking to DynamoDB. Instead, it just ' +
+    'tracks the changes it would have made locally.'
+
+  opts.on('--dry-run', doc) do
+    general_opts[:dry_run] = true
+  end
+
+  doc = "Sets a minimum value for throughputs to be set to. " +
+    "Defaults to 10."
+
+  opts.on('--minimum-throughput VALUE', Float, doc) do |value|
+    if value < 1.0
+      STDERR.puts "Cannot set minimum throughput to less than 1."
+      exit 1
+    end
+
+    general_opts[:minimum_throughput] = value
+  end
+
+  doc = "Sets a maximum value for throughputs to be set to. " +
+    "Defaults to 20,000."
+
+  opts.on('--maximum-throughput VALUE', Float, doc) do |value|
+    general_opts[:maximum_throughput] = value
+  end
+
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end.parse!
+
+ruleset = ARGV.shift
+tables  = ARGV
+
+if tables.empty? or ruleset.nil?
+  STDERR.puts "Usage: dynamo-autoscale ruleset table_name [another_table_name ... ]"
+  exit 1
+end
+
+if actioner_opts[:flush_after] and actioner_opts[:group_downscales].nil?
+  STDERR.puts "Cannot specify a flush_after value with setting --group-downscales."
+  exit 1
+end
+
+require_relative '../config/environment/common'
+include DynamoAutoscale
+extend  DynamoAutoscale
+
+dynamo = AWS::DynamoDB.new
+tables.select! do |table_name|
+  if dynamo.tables[table_name].exists?
+    true
+  else
+    logger.error "Table #{table_name} does not exist inside your DynamoDB."
+    false
+  end
+end
+
+if tables.empty?
+  STDERR.puts "No valid tables specified."
+  exit 1
+end
+
+poller_opts = { tables: tables }
+
+if general_opts[:dry_run]
+  poller_opts[:filters] = LocalActioner.faux_provisioning_filters
+end
+
+DynamoAutoscale.rules          = RuleSet.new(ruleset)
+DynamoAutoscale.dispatcher     = Dispatcher.new
+DynamoAutoscale.poller         = CWPoller.new(poller_opts)
+DynamoAutoscale.actioner_class = general_opts[:dry_run] ? LocalActioner : DynamoActioner
+DynamoAutoscale.actioner_opts  = actioner_opts
+
+if general_opts[:minimum_throughput]
+  Actioner.minimum_throughput = general_opts[:minimum_throughput]
+end
+
+if general_opts[:maximum_throughput]
+  Actioner.maximum_throughput = general_opts[:maximum_throughput]
+end
+
+Signal.trap("USR1") do
+  logger.info "[signal] Caught SIGUSR1. Dumping CSV for all tables in #{Dir.pwd}"
+
+  DynamoAutoscale.tables.each do |name, table|
+    table.to_csv! path: File.join(Dir.pwd, "#{table.name}.csv")
+  end
+end
+
+Signal.trap("USR2") do
+  logger.info "[signal] Caught SIGUSR2. Dumping graphs for all tables in #{Dir.pwd}"
+
+  DynamoAutoscale.tables.each do |name, table|
+    table.graph! path: File.join(Dir.pwd, "#{table.name}.png")
+  end
+end
+
+DynamoAutoscale.poller.run