heroku-vector 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 00d6ccf09122d48c58eabcc34fb1a8bb16a3696e
+  data.tar.gz: be88b82fe45f6764c76dd0d35e7fb020e1223a92
+SHA512:
+  metadata.gz: 20ddb99b29041e06d78111a9dd8a3092160855bf1b1ff9e640228caae7964a6f46198f709bf68cf4cc7a105429f8f171bebd4872463764f91506b898d5f1260f
+  data.tar.gz: 5c0052f0b3e3b3f227ee0124ed971a8ff341e66a8aec3f92c6632d16923a51fb1342e447a1a56c2d1deb00a51811e279f6d517a524ff435421f742de6ba5389f

data/.gitignore

@@ -0,0 +1,37 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+.env
+config.rb
+

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Releases / Changes
+
+## 0.0.2
+
+* Config/usability fixes
+* Awesome Ascii Logo
+
+## 0.0.1
+
+* Initial gem release

data/Gemfile

@@ -0,0 +1,4 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,95 @@
+PATH
+  remote: .
+  specs:
+    heroku-vector (0.0.2)
+      activeresource
+      dotenv
+      eventmachine
+      heroku-api
+      newrelic_api
+      redis
+      redis-namespace
+      sidekiq
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.1.6)
+      activesupport (= 4.1.6)
+      builder (~> 3.1)
+    activeresource (4.0.0)
+      activemodel (~> 4.0)
+      activesupport (~> 4.0)
+      rails-observers (~> 0.1.1)
+    activesupport (4.1.6)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    addressable (2.3.6)
+    builder (3.2.2)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    coderay (1.1.0)
+    connection_pool (2.0.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    dotenv (0.11.1)
+      dotenv-deployment (~> 0.0.2)
+    dotenv-deployment (0.0.2)
+    eventmachine (1.0.3)
+    excon (0.39.5)
+    heroku-api (0.3.19)
+      excon (~> 0.38)
+      multi_json (~> 1.8)
+    i18n (0.6.11)
+    json (1.8.1)
+    metaclass (0.0.4)
+    method_source (0.8.2)
+    minitest (5.4.0)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    multi_json (1.10.1)
+    newrelic_api (1.2.4)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rails-observers (0.1.2)
+      activemodel (~> 4.0)
+    rake (10.3.2)
+    redis (3.1.0)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
+    safe_yaml (1.0.3)
+    sidekiq (3.2.5)
+      celluloid (= 0.15.2)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    slop (3.6.0)
+    thread_safe (0.3.4)
+    timecop (0.7.1)
+    timers (1.1.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vcr (2.9.2)
+    webmock (1.18.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  heroku-vector!
+  minitest
+  mocha
+  pry
+  rake
+  timecop
+  vcr
+  webmock

data/README.md

@@ -0,0 +1,171 @@
+# Heroku Vector
+
+```bash
+  ___ ___                           __                  
+ /   |   \   ____  _______   ____  |  | __ __ __        
+/    ~    \_/ __ \ \_  __ \ /  _ \ |  |/ /|  |  \       
+\    Y    /\  ___/  |  | \/(  <_> )|    < |  |  /       
+ \___|_  /  \___  > |__|    \____/ |__|_ \|____/        
+      ____   ____/                 __   \/              
+      \   \ /   /  ____    ____  _/  |_   ____  _______ 
+       \   Y   / _/ __ \ _/ ___\ \   __\ /  _ \ \_  __ \
+        \     /  \  ___/ \  \___  |  |  (  <_> ) |  | \/
+         \___/    \___  > \___  > |__|   \____/  |__|   
+                      \/      \/                        
+```
+
+Simple, linear auto scaling for Heroku dynos.
+
+Heroku Vector runs as a multi-threaded process that samples production metrics and linearly scales Heroku dynos up or down as they change.
+
+Web Dynos can be scaled using the amount of traffic (RPM) from NewRelic.  Sidekiq dynos can be scaled based on how many worker threads are busy.
+
+## Installation
+
+Install this gem:
+
+    $ gem install heroku_vector
+
+And then run the scaler proces:
+
+    $ heroku_vector --help
+
+## Configuration
+
+This auto scaler will sample metrics and use the metric values to scale different types of Heroku Dynos up or down.
+
+```ruby 
+
+# Example config.rb file
+HerokuVector.configure do |config|
+
+  # Scale 1 web dyno for each 300rpm of traffic
+  config.add_dyno_scaler('web', {
+    source:         HerokuVector::Source::NewRelic,
+    period:         60,
+    min_dynos:      1,
+    max_dynos:      10,
+    min_value:      100,
+    max_value:      300
+  })
+
+  # Manually specify app config in config.rb like so
+  # Alternately, use environment variables
+  config.newrelic_api_key = '222222222222222'
+
+end
+```
+
+Application config can be wired up in the config.rb file, or passed in via the system Environment or an Environment file:
+
+```bash
+HEROKU_APP_NAME=your-app-name
+HEROKU_API_KEY=1111111111111111
+REDIS_URL=redis://redis.yourcompany.com/
+SIDEKIQ_REDIS_NAMESPACE=sidekiq
+NEWRELIC_API_KEY=222222222222222
+```
+
+## heroku_vector daemon:
+```bash
+[master ~/src/heroku-vector]$> ./bin/heroku_vector --help
+heroku_vector: auto-scale dynos on Heroku
+
+  Usage: heroku_vector [options]
+         heroku_vector -s
+
+    -s, --sample                     Sample values and exit
+    -d, --daemonize                  Daemonize process
+    -e, --envfile PATH               Environment file (default: .env)
+    -c, --config PATH                Config file (default: config.rb)
+    -p, --pidfile PATH               Daemon pid file (default: heroku_vector.pid)
+    -x, --loglevel LEVEL             Logging level [fatal/warn/info/debug] (default is info)
+    -l, --logfile PATH               Logfile path for daemon
+    -h, --help                       Show this message
+```
+
+Once you've configured your API keys and host names, try taking a sample from all your sources:
+
+```bash
+heroku_vector: {:daemonize=>false, :envfile=>"/Users/wpeterson/src/heroku-vector/.env", :config=>"/Users/wpeterson/src/heroku-vector/config.rb", :sample=>true}
+HerokuVector::Source::NewRelic: 8490.0 RPM
+HerokuVector::Source::Sidekiq: 23 busy threads
+```
+
+## Logging and Debugging
+
+Heroku Vector logs either to `STDOUT` or a logfile with useful information about state changes and scaling events.
+
+```bash
+2014-09-12T16:05:48.473Z INFO: Loading config from '/home/ubuntu/polar-auto-scale/config.rb'
+2014-09-12T16:05:48.474Z INFO: Loading Scaler: web, {:source=>HerokuVector::Source::NewRelic, :period=>60, :min_dynos=>1, :max_dynos=>4, :min_value=>1000, :max_value=>3000}
+2014-09-12T16:05:48.474Z INFO: Loading Scaler: worker, {:source=>HerokuVector::Source::Sidekiq, :period=>5, :min_dynos=>1, :max_dynos=>10, :min_value=>0.5, :max_value=>3, :scale_up_by=>3, :scale_down_by=>1}
+2014-09-12T16:45:01.742Z INFO: Heroku.scale_dynos(worker, 4)
+2014-09-12T16:50:03.164Z INFO: worker: 4 dynos - 0.4 busy threads below 2.0 - scaling down
+2014-09-12T16:50:03.247Z INFO: Heroku.scale_dynos(worker, 3)
+2014-09-12T16:55:04.559Z INFO: worker: 3 dynos - 0.2 busy threads below 1.5 - scaling down
+2014-09-12T16:55:04.646Z INFO: Heroku.scale_dynos(worker, 2)
+2014-09-12T17:00:08.391Z INFO: worker: 2 dynos - 0.9 busy threads below 1.0 - scaling down
+2014-09-12T17:00:08.492Z INFO: Heroku.scale_dynos(worker, 1)
+```
+
+If you're debugging a problem, you can turn on verbose logging by setting the `debug` log level:
+
+    $ heroku_vector -x debug
+
+## Architecture
+
+![Architecture](https://dl.dropboxusercontent.com/s/a133uy8e0ohwp9t/Architecture.png?dl=0)
+
+The auto-scaler runs as a single process, either interactively or as a daemon (`ProcessManager`).  Within that process, the `Worker` spawn an EventMachine event loop and runs each `DynoScaler` in it's own thread.  Periodically, each `DynoScaler` will sample data and evaluate the scale of your dynos.  When the scale of your dynos doesn't match your traffic, the `DynoScaler` will use the Heroku API to scale your dynos up or down.
+
+## Adding a new Source
+
+Data is sampled from generic `Source` classes in the Ruby namespace `HerokuVector::Source` and wired up in your configuration.  Source objects have a simple contract:
+
+* `#sample` - returns a numeric value for this source at this time
+* `#units` - string that describes the data, like 'RPM' for NewRelic
+
+You can define your own `Source` classes and then reference them within your config:
+
+```ruby
+# my_data_source.rb
+module HerokuVector::Source
+  class MyDataSource
+    def sample
+      # Always returns 1, simple test case
+      1
+    end
+
+    def units
+      'foos'
+    end
+  end
+end
+
+# Wire Up Your Source in config.rb
+require 'path/to/my_data_source'
+
+HerokuVector.configure do |config|
+
+  # Scale 1 web dyno for each 300rpm of traffic
+  config.add_dyno_scaler('web', {
+    source:         HerokuVector::Source::MyDataSource,
+    period:         1
+  })
+
+```
+
+## Contributors: :heart:
+
+* Your Name Could Go Here
+
+## Contributing
+
+1. Fork it
+2. Bundle Install (`bundle install`)
+3. Run the Tests (`rake test`)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/bin/heroku_vector

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'dotenv'
+require 'optparse'
+
+options = {
+  :daemonize  => false,
+  :envfile    => File.expand_path( File.dirname(__FILE__)+'/../.env' ),
+  :config     => File.expand_path( File.dirname(__FILE__)+'/../config.rb' ),
+  :sample     => false
+}
+
+def show_usage_and_exit
+  puts @parser
+  exit
+end
+
+@parser = OptionParser.new do |opts|
+  opts.banner = <<EOL
+heroku_vector: auto-scale dynos on Heroku
+
+  Usage: heroku_vector [options]
+         heroku_vector -s
+
+EOL
+
+  opts.on('-s', '--sample', "Sample values and exit") {|v| options[:sample] = true }
+  opts.on('-d', '--daemonize', "Daemonize process") {|v| options[:daemonize] = true }
+  opts.on('-e', '--envfile PATH', "Environment file (default: .env)") {|v| options[:envfile] = v }
+  opts.on('-c', '--config PATH', "Config file (default: config.rb)") {|v| options[:config] = v }
+  opts.on('-p', '--pidfile PATH', "Daemon pid file (default: heroku_vector.pid)") {|v| options[:pidfile] = v }
+  opts.on('-x', '--loglevel LEVEL', "Logging level [fatal/warn/info/debug] (default is info)") {|v| options[:loglevel] = v }
+  opts.on('-l', '--logfile PATH', "Logfile path for daemon") {|v| options[:logfile] = v }
+  opts.on_tail("-h", "--help", "Show this message") { show_usage_and_exit }
+end
+@parser.parse!(ARGV)
+
+if options[:envfile] && File.exist?(options[:envfile])
+  env = Dotenv.load(options[:envfile])
+end
+
+# Write a default pidfile when daemonized
+if options[:daemonize]
+  options[:pidfile] ||= File.expand_path( File.dirname(__FILE__)+'/../heroku_vector.pid' )
+  options[:logfile] ||= File.expand_path( File.dirname(__FILE__)+'/../heroku_vector.log' )
+end
+
+puts "heroku_vector: #{options.inspect}"
+
+# Load heroku_vector after all options parses and environment loaded
+require 'heroku_vector'
+
+if options[:loglevel]
+  level = Logger.const_get(options[:loglevel].upcase) rescue nil
+  HerokuVector.logger.level = level if level
+end
+
+if options[:sample]
+  # Print out a sample value for each valid Source class
+  HerokuVector::Source.constants.each do |constant|
+    clazz = HerokuVector::Source.const_get(constant)
+    next unless clazz.class == Class
+
+    source = clazz.new
+    puts "#{clazz}: #{source.sample} #{source.unit}"
+  end
+
+  exit 0
+end
+
+logo_ascii = <<EOL
+
+  ___ ___                           __
+ /   |   \\   ____  _______   ____  |  | __ __ __
+/    ~    \\_/ __ \\ \\_  __ \\ /  _ \\ |  |/ /|  |  \\
+\\    Y    /\\  ___/  |  | \\/(  <_> )|    < |  |  /
+ \\___|_  /  \\___  > |__|    \\____/ |__|_ \\|____/
+      ____   ____/                 __   \\/
+      \\   \\ /   /  ____    ____  _/  |_   ____  _______
+       \\   Y   / _/ __ \\ _/ ___\\ \\   __\\ /  _ \\ \\_  __ \\
+        \\     /  \\  ___/ \\  \\___  |  |  (  <_> ) |  | \\/
+         \\___/    \\___  > \\___  > |__|   \\____/  |__|
+                      \\/      \\/
+EOL
+puts logo_ascii
+
+manager = HerokuVector::ProcessManager.new(options)
+manager.start

data/config.rb.example

@@ -0,0 +1,28 @@
+# Example config.rb file
+# Copy this to config.rb and edit for your configuration
+
+HerokuVector.configure do |config|
+
+  # Scale 1 web dyno for each 300rpm of traffic
+  config.add_dyno_scaler('web', {
+    source:         HerokuVector::Source::NewRelic,
+    period:         60,
+    min_dynos:      1,
+    max_dynos:      10,
+    min_value:      100,
+    max_value:      300
+  })
+
+  # Scale up Sidekiq workers when there are 6 or more busy threads per worker
+  # Scale up two dynos at a time, but scale down 1 dyno at a time
+  config.add_dyno_scaler('worker', {
+    source:         HerokuVector::Source::Sidekiq,
+    period:         5,
+    min_dynos:      1,
+    max_dynos:      10,
+    min_value:      0.5,
+    max_value:      6,
+    scale_up_by:    2,
+    scale_down_by:  1
+  })
+end