whacamole 0.0.1 → 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+*.swp
+whacamole*.gem

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gemspec :name => 'whacamole'
+

data/Gemfile.lock

@@ -0,0 +1,26 @@
+PATH
+  remote: .
+  specs:
+    whacamole (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.4)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.2)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 10.1)
+  rspec (~> 2.11)
+  whacamole!

data/README.md

@@ -0,0 +1,64 @@
+whacamole
+=========
+
+[![Build Status](https://travis-ci.org/arches/whacamole.png)](https://travis-ci.org/arches/whacamole)
+
+Whacamole keeps track of your Heroku dynos' memory usage and restarts large dynos before they start
+swapping to disk (aka get super slow).
+
+Here’s what Heroku says about dyno memory usage:
+
+> Dynos are available in 1X or 2X sizes and are allocated 512MB or 1024MB respectively.
+>
+> Dynos whose processes exceed their memory quota are identified by an R14 error in the logs. This doesn’t terminate the process, but it does warn of deteriorating application conditions: memory used above quota will swap out to disk, which substantially degrades dyno performance.
+>
+> If the memory size keeps growing until it reaches three times its quota, the dyno manager will restart your dyno with an R15 error.
+>
+> - From https://devcenter.heroku.com/articles/dynos on 8/8/13
+
+Heroku dynos swap to disk for up to 3GB. That is not good and that is the problem whacamole addresses.
+
+# Usage
+
+Enable log-runtime-metrics on your heroku app:
+
+```bash
+$ heroku labs:enable log-runtime-metrics --app YOUR_APP_NAME
+```
+
+Add whacamole to your gemfile:
+
+```ruby
+gem 'whacamole'
+```
+
+Create a config file with your app info. Personally I put it in my Rails app at config/whacamole.rb. The
+most important parts are your app name and your Heroku api token (which can be found by running `heroku auth:token`
+on the command line).
+
+```ruby
+Whacamole.configure("HEROKU APP NAME") do |config|
+  config.api_token = ENV['HEROKU_API_TOKEN'] # you could also paste your token in here as a string
+end
+
+# you can monitor multiple apps at once, just add more configure blocks
+Whacamole.configure("ANOTHER HEROKU APP") do |config|
+  config.api_token = ENV['HEROKU_API_TOKEN'] # you could also paste your token in here as a string
+end
+```
+
+Add whacamole to your Procfile, specifying the config file you created:
+
+```ruby
+whacamole: bundle exec whacamole -c ./config/whacamole.rb
+```
+
+Start foreman, and you're done!
+
+```bash
+# locally
+$ foreman start whacamole
+
+# on heroku
+$ heroku ps:scale whacamole=1 --app YOUR_APP_NAME
+```

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => [:spec]
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+end
+

data/bin/whacamole

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'whacamole'
+$stdout.sync = true
+
+if ARGV[0] == "-c"
+  require ARGV[1]
+
+  Whacamole.monitor
+else
+  puts "No Whacamole config specified. Use `whacamole -c path/to/config`\nSee http://github.com/arches/whacamole for usage."
+end
+

data/lib/whacamole.rb

@@ -0,0 +1,30 @@
+require 'whacamole/config'
+require 'whacamole/events'
+require 'whacamole/heroku_wrapper'
+require 'whacamole/stream'
+
+module Whacamole
+
+  @@config = {}
+
+  def self.configure(app_name)
+    @@config[app_name.to_s] ||= Config.new(app_name)
+    yield @@config[app_name.to_s]
+  end
+
+  def self.monitor
+    threads = []
+    @@config.each do |app_name, config|
+      threads << Thread.new do
+        heroku = HerokuWrapper.new(app_name, config.api_token)
+
+        while true
+          stream_url = heroku.create_log_session
+          Stream.new(stream_url, heroku, &config.event_handler).watch
+        end
+      end
+    end
+    threads.collect(&:join)
+  end
+end
+

data/lib/whacamole/config.rb

@@ -0,0 +1,11 @@
+module Whacamole
+  class Config
+    attr_accessor :app_name, :api_token, :event_handler
+
+    def initialize(app_name)
+      self.app_name = app_name
+      self.event_handler ||= lambda { |e| puts e.inspect.to_s }
+    end
+  end
+end
+

data/lib/whacamole/events.rb

@@ -0,0 +1,26 @@
+module Whacamole
+  module Events
+    class Event
+      attr_accessor :process
+
+      def initialize(attributes={})
+        attributes.each do |k,v|
+          self.send("#{k}=", v)
+        end
+      end
+    end
+
+    class DynoSize < Event
+      attr_accessor :units
+      attr_reader :size
+
+      def size=(size)
+        @size = size.to_f
+      end
+    end
+
+    class DynoRestart < Event
+    end
+  end
+end
+

data/lib/whacamole/heroku_wrapper.rb

@@ -0,0 +1,72 @@
+require 'base64'
+require 'net/http'
+require 'json'
+
+module Whacamole
+  class HerokuWrapper
+    attr_accessor :api_token, :app_name
+
+    RESTART_RATE_LIMIT = 30*60
+
+    def initialize(app_name, api_token)
+      self.app_name = app_name
+      self.api_token = api_token
+    end
+
+    def create_log_session
+      uri = URI(log_sessions_url)
+      req = Net::HTTP::Post.new(uri.path)
+      req['Authorization'] = authorization
+      req['Content-type'] = content_type
+      req['Accept'] = accept
+      req.set_form_data({'tail' => true})
+      res = Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) {|http| http.request(req)}
+      JSON.parse(res.body)['logplex_url']
+    end
+
+    def authorization
+      "Basic " + Base64.encode64(":#{api_token}").gsub("\n", '')
+    end
+
+    def restart(process)
+      return false if recently_restarted?(process)
+
+      uri = URI(dyno_url(process))
+      req = Net::HTTP::Delete.new(uri.path)
+      req['Authorization'] = authorization
+      req['Content-type'] = content_type
+      req['Accept'] = accept
+      res = Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) {|http| http.request(req)}
+
+      restarts[process] = Time.now
+
+      true
+    end
+
+    def recently_restarted?(process)
+      restarts[process] > (Time.now - RESTART_RATE_LIMIT)
+    end
+
+    private
+    def content_type
+      "application/json"
+    end
+
+    def accept
+      "application/vnd.heroku+json; version=3"
+    end
+
+    def log_sessions_url
+      "https://api.heroku.com/apps/#{app_name}/log-sessions"
+    end
+
+    def dyno_url(process)
+      "https://api.heroku.com/apps/#{app_name}/dynos/#{process}"
+    end
+
+    def restarts
+      @restarts ||= Hash.new { Time.now - RESTART_RATE_LIMIT*2 }
+    end
+  end
+end
+

data/lib/whacamole/stream.rb

@@ -0,0 +1,81 @@
+require 'net/http'
+
+module Whacamole
+
+  class Stream
+
+    RESTART_THRESHOLD = 1000
+
+    def initialize(url, restart_handler, &blk)
+      @url = url
+      @restart_handler = restart_handler
+      @event_handler = blk
+    end
+
+    def watch
+      uri = URI(url)
+      Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
+        request = Net::HTTP::Get.new(uri.request_uri)
+        http.request(request) do |response|
+          response.read_body do |chunk|
+            dispatch_handlers(chunk)
+          end
+        end
+      end
+    end
+
+    def dispatch_handlers(chunk)
+      memory_size_from_chunk(chunk).each do |dyno, size|
+        event = Events::DynoSize.new({:process => dyno, :size => size, :units => "MB"})
+        event_handler.call(event)
+
+        restart(event.process) if restart_necessary?(event)
+      end
+
+      # TODO: handle R14 errors here also
+    end
+
+    private
+    def restart(process)
+      restarted = restart_handler.restart(process)
+
+      if restarted
+        event_handler.call( Events::DynoRestart.new({:process => process}) )
+      end
+    end
+
+    def restart_necessary?(event)
+      event.size > restart_threshold
+    end
+
+    def memory_size_from_chunk(chunk)
+      sizes = []
+      chunk.split("\n").select{|line| line.include? "sample#memory_total"}.each do |line|
+        dyno = line.match(/web\.\d+/)
+        next unless dyno
+        size = line.match(/sample#memory_total=([\d\.]+)/)
+        sizes << [dyno[0], size[1]]
+      end
+      sizes
+    end
+
+    def url
+      @url
+    end
+
+    def event_handler
+      @event_handler
+    end
+
+    def restart_handler
+      @restart_handler
+    end
+
+    def restart_threshold
+      RESTART_THRESHOLD
+    end
+  end
+end
+
+
+

data/lib/whacamole/version.rb

@@ -1,4 +1,4 @@
 module Whacamole
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end
 

data/spec/config_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Whacamole::Config do
+  describe "initialization" do
+    it "sets the app name" do
+      c = Whacamole::Config.new("production")
+      c.app_name.should == "production"
+    end
+  end
+end
+

data/spec/events_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe "DynoSize" do
+  let (:e) { Whacamole::Events::DynoSize.new }
+
+  describe "setting size" do
+    it "converts strings to floats" do
+      e.size = "766.65"
+      e.size.should == 766.65
+    end
+  end
+
+  describe "setting units" do
+    it "stores the units on the object" do
+      e.units = "MB"
+      e.units.should == "MB"
+    end
+  end
+
+  describe "setting process" do
+    it "stores the process on the object" do
+      e.process = "web.2"
+      e.process.should == "web.2"
+    end
+  end
+
+  describe "initialization" do
+    it "sets the variables from the input hash" do
+      e = Whacamole::Events::DynoSize.new({:size => "766.65", :units => "MB", :process => "web.2"})
+      e.size.should == 766.65
+      e.units.should == "MB"
+      e.process.should == "web.2"
+    end
+  end
+end
+

data/spec/heroku_wrapper_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+# TODO: use VCR instead of these method expectations
+describe Whacamole::HerokuWrapper do
+  let(:h) { Whacamole::HerokuWrapper.new("staging", "foobar") }
+
+  describe "authorization" do
+    it "base64-encodes the api token and a preceding colon" do
+      h.authorization.should == "Basic OmZvb2Jhcg=="
+    end
+  end
+
+  describe "create_log_session" do
+    it "executes a request using the default headers" do
+      req = {}
+      Net::HTTP::Post.should_receive(:new).with("/apps/staging/log-sessions") { req }
+      req.should_receive(:[]=).with("Authorization", h.authorization)
+      req.should_receive(:[]=).with("Content-type", "application/json")
+      req.should_receive(:[]=).with("Accept", "application/vnd.heroku+json; version=3")
+      req.should_receive(:set_form_data).with({'tail' => true})
+      Net::HTTP.should_receive(:start).with("api.heroku.com", 443, use_ssl: true).and_return(OpenStruct.new(:body => "{\"logplex_url\": \"https://api.heroku.com/log/session/url\"}"))
+      h.create_log_session.should == "https://api.heroku.com/log/session/url"
+    end
+  end
+
+  describe "restart" do
+    it "executes a request using the default headers" do
+      req = {}
+      Net::HTTP::Delete.should_receive(:new).with("/apps/staging/dynos/web.3") { req }
+      req.should_receive(:[]=).with("Authorization", h.authorization)
+      req.should_receive(:[]=).with("Content-type", "application/json")
+      req.should_receive(:[]=).with("Accept", "application/vnd.heroku+json; version=3")
+      Net::HTTP.should_receive(:start).with("api.heroku.com", 443, use_ssl: true)
+      h.restart("web.3").should be_true
+    end
+
+    it "respects the rate limit" do
+      Net::HTTP::Delete.should_not_receive(:new)
+
+      h.stub(:recently_restarted? => true)
+      h.restart("web.2").should be_false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+gem 'rspec'
+require 'ostruct'
+require 'whacamole'
+
+RSpec.configure do |c|
+  c.color = true
+  c.filter_run :focus => true
+  c.run_all_when_everything_filtered = true
+end
+

data/spec/stream_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+class EventHandler
+  attr_accessor :events
+
+  def process(event)
+    events << event
+  end
+
+  def events
+    @events ||= []
+  end
+end
+
+class RestartHandler
+  def restart(process)
+    true
+  end
+end
+
+describe Whacamole::Stream do
+  let(:eh) { EventHandler.new }
+  let(:restart_handler) { RestartHandler.new }
+  let(:stream) do
+    Whacamole::Stream.new("https://api.heroku.com/path/to/stream/stream", restart_handler) do |event|
+      eh.process(event)
+    end
+  end
+
+  describe "stream" do
+    it "opens the url for streaming" do
+      stream.watch
+    end
+  end
+
+  describe "handle_chunk" do
+    context "when memory usage is present" do
+      it "surfaces the memory usage" do
+        stream.dispatch_handlers <<-CHUNK
+          2013-08-22T16:39:22.208103+00:00 heroku[router]: at=info method=GET path=/favicon.ico host=aisle50.com fwd="205.159.94.63" dyno=web.3 connect=1ms service=20ms status=200 bytes=894
+          2013-08-22T16:39:22.224847+00:00 heroku[router]: at=info method=GET path=/ host=www.aisle50.com fwd="119.63.193.132" dyno=web.3 connect=1ms service=5ms status=301 bytes=0
+          2013-08-22T16:39:22.919300+00:00 heroku[web.2]: source=web.2 dyno=heroku.772639.a334caa8-736c-48b3-bac2-d366f75d7fa0 sample#load_avg_1m=0.20 sample#load_avg_5m=0.33 sample#load_avg_15m=0.38
+          2013-08-22T16:39:22.919536+00:00 heroku[web.2]: source=web.2 dyno=heroku.772639.a334caa8-736c-48b3-bac2-d366f75d7fa0 sample#memory_total=581.95MB sample#memory_rss=581.75MB sample#memory_cache=0.16MB sample#memory_swap=0.03MB sample#memory_pgpgin=0pages sample#memory_pgpgout=179329pages
+          2013-08-22T16:39:22.919773+00:00 heroku[web.2]: source=web.2 dyno=heroku.772639.a334caa8-736c-48b3-bac2-d366f75d7fa0 sample#diskmbytes=0MB
+          2013-08-22T16:39:23.045250+00:00 heroku[web.1]: source=web.1 dyno=heroku.772639.4c9dcf54-f339-4d81-9756-8dad47f178a4 sample#load_avg_1m=0.24 sample#load_avg_5m=0.59
+          2013-08-22T16:39:23.045521+00:00 heroku[web.90]: source=web.1 dyno=heroku.772639.4c9dcf54-f339-4d81-9756-8dad47f178a4 sample#memory_total=66MB sample#memory_rss=471.21MB sample#memory_cache=0.05MB sample#memory_swap=0.02MB sample#memory_pgpgin=0pages sample#memory_pgpgout=145277pages
+          2013-08-22T16:39:23.045789+00:00 heroku[web.1]: source=web.1 dyno=heroku.772639.4c9dcf54-f339-4d81-9756-8dad47f178a4 sample#diskmbytes=0MB
+          2013-08-22T16:39:23.364649+00:00 heroku[worker.1]: source=worker.1 dyno=heroku.772639.ae391b5d-e776-43f9-b056-360912563d61 sample#load_avg_1m=0.00 sample#load_avg_5m=0.01 sample#load_avg_15m=0.02
+        CHUNK
+
+        eh.events.length.should == 2
+
+        eh.events.first.should be_a Whacamole::Events::DynoSize
+        eh.events.first.size.should == 581.95
+        eh.events.first.units.should == "MB"
+        eh.events.first.process.should == "web.2"
+
+        eh.events.last.should be_a Whacamole::Events::DynoSize
+        eh.events.last.size.should == 66.0
+        eh.events.last.units.should == "MB"
+        eh.events.last.process.should == "web.90"
+      end
+    end
+
+    context "when memory usages is over the threshold" do
+      it "kicks off a restart" do
+        restart_handler.should_receive(:restart).with("web.2")
+        stream.dispatch_handlers <<-CHUNK
+          2013-08-22T16:39:22.919536+00:00 heroku[web.2]: source=web.2 dyno=heroku.772639.a334caa8-736c-48b3-bac2-d366f75d7fa0 sample#memory_total=1001MB sample#memory_rss=581.75MB sample#memory_cache=0.16MB sample#memory_swap=0.03MB sample#memory_pgpgin=0pages sample#memory_pgpgout=179329pages
+        CHUNK
+      end
+
+      it "surfaces the restart" do
+        stream.dispatch_handlers <<-CHUNK
+          2013-08-22T16:39:22.919536+00:00 heroku[web.2]: source=web.2 dyno=heroku.772639.a334caa8-736c-48b3-bac2-d366f75d7fa0 sample#memory_total=1001MB sample#memory_rss=581.75MB sample#memory_cache=0.16MB sample#memory_swap=0.03MB sample#memory_pgpgin=0pages sample#memory_pgpgout=179329pages
+        CHUNK
+
+        restart = eh.events.last
+        restart.should be_a Whacamole::Events::DynoRestart
+        restart.process.should == "web.2"
+      end
+    end
+  end
+end
+
+

data/spec/whacamole_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Whacamole do
+  describe "configure" do
+    it "returns unique config objects by app name" do
+      prod_config = nil
+      Whacamole.configure("production") do |config|
+        prod_config = config
+      end
+
+      staging_config = nil
+      Whacamole.configure("staging") do |config|
+        staging_config = config
+      end
+
+      staging_config.app_name.should == "staging"
+      prod_config.app_name.should == "production"
+
+      prod_config.should_not == staging_config
+    end
+
+    it "returns the same config object if asked a second time" do
+      Whacamole.configure("production") do |config|
+        config.api_token = "prod token"
+      end
+
+      Whacamole.configure("production") do |config|
+        config.api_token.should == "prod token"
+      end
+    end
+  end
+end
+

data/whacamole.gemspec

@@ -18,7 +18,8 @@ Gem::Specification.new do |gem|
   gem.test_files          = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths       = ["lib"]
 
-  gem.add_development_dependency 'cucumber', '~> 1.2.1'
-  gem.add_development_dependency 'rspec', '~> 2.11.0'
-  gem.add_development_dependency 'rake', '~> 0.9.2'
+  gem.executables         = ['whacamole']
+
+  gem.add_development_dependency 'rspec', '~> 2.11'
+  gem.add_development_dependency 'rake', '~> 10.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whacamole
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-21 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: cucumber
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -34,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.11.0
+        version: '2.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.11.0
+        version: '2.11'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '10.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,14 +42,33 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '10.1'
 description: Whacamole
 email: archslide@gmail.com
-executables: []
+executables:
+- whacamole
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/whacamole
+- lib/whacamole.rb
+- lib/whacamole/config.rb
+- lib/whacamole/events.rb
+- lib/whacamole/heroku_wrapper.rb
+- lib/whacamole/stream.rb
 - lib/whacamole/version.rb
+- spec/config_spec.rb
+- spec/events_spec.rb
+- spec/heroku_wrapper_spec.rb
+- spec/spec_helper.rb
+- spec/stream_spec.rb
+- spec/whacamole_spec.rb
 - whacamole.gemspec
 homepage: http://github.com/arches/whacamole
 licenses:
@@ -92,4 +95,10 @@ rubygems_version: 1.8.19
 signing_key: 
 specification_version: 3
 summary: ''
-test_files: []
+test_files:
+- spec/config_spec.rb
+- spec/events_spec.rb
+- spec/heroku_wrapper_spec.rb
+- spec/spec_helper.rb
+- spec/stream_spec.rb
+- spec/whacamole_spec.rb