litmus_paper 0.0.3

data/lib/litmus_paper.rb

@@ -0,0 +1,51 @@
+require 'pathname'
+require 'net/http'
+require 'uri'
+require 'forwardable'
+
+require 'sinatra/base'
+require 'facter'
+require 'syslog_logger'
+
+require 'facts/loadaverage'
+
+require 'litmus_paper/app'
+require 'litmus_paper/configuration'
+require 'litmus_paper/dependency/http'
+require 'litmus_paper/dependency/tcp'
+require 'litmus_paper/health'
+require 'litmus_paper/forced_health'
+require 'litmus_paper/logger'
+require 'litmus_paper/metric/available_memory'
+require 'litmus_paper/metric/cpu_load'
+require 'litmus_paper/service'
+require 'litmus_paper/status_file'
+
+module LitmusPaper
+  class << self
+    attr_reader :services, :config_dir
+  end
+
+  def self.configure(filename)
+    @config_file = filename
+
+    begin
+      @services = LitmusPaper::Configuration.new(filename).evaluate
+    rescue Exception
+    end
+  end
+
+  def self.config_dir=(path)
+    @config_dir = Pathname.new(path)
+  end
+
+  def self.reload
+    configure(@config_file)
+  end
+
+  def self.reset
+    @services = {}
+  end
+end
+
+Signal.trap("HUP") { LitmusPaper.reload }

data/litmus_paper.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/litmus_paper/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Braintreeps"]
+  gem.email         = ["code@getbraintree.com"]
+  gem.description   = %q{Backend health tester for HA Services}
+  gem.summary       = %q{Backend health tester for HA Services}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "litmus_paper"
+  gem.require_paths = ["lib"]
+  gem.version       = LitmusPaper::VERSION
+
+  gem.add_dependency "sinatra", "~> 1.3.2"
+  gem.add_dependency "facter", "~> 1.6.7"
+  gem.add_dependency "SyslogLogger", "1.4.1"
+
+  gem.add_development_dependency "rspec", "2.9.0"
+  gem.add_development_dependency "rack-test", "0.6.1"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rake_commit", "0.13"
+end

data/spec/litmus_paper/app_spec.rb

@@ -0,0 +1,246 @@
+require 'spec_helper'
+
+describe LitmusPaper::App do
+  def app
+    LitmusPaper::App
+  end
+
+  describe "GET /" do
+    it "returns the list of services litmus monitors" do
+      LitmusPaper.services['test'] = LitmusPaper::Service.new('test')
+      LitmusPaper.services['another'] = LitmusPaper::Service.new('another')
+
+      get "/"
+
+      last_response.status.should == 200
+      last_response.body.should include('test')
+      last_response.body.should include('another')
+    end
+  end
+
+  describe "POST /force/*" do
+    it "creates a global upfile" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/up", :reason => "up for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 200
+      last_response.body.should match(/up for testing/)
+    end
+
+    it "creates a global downfile" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/down", :reason => "down for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 503
+      last_response.body.should match(/down for testing/)
+    end
+
+    it "creates a service specific upfile" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/up/test", :reason => "up for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 200
+      last_response.body.should match(/up for testing/)
+    end
+  end
+
+  describe "DELETE /force/*" do
+    it "removes the global upfile" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/up", :reason => "up for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 200
+
+      delete "/force/up"
+      last_response.status.should == 200
+
+      get "/test/status"
+      last_response.status.should == 503
+      last_response.body.should_not match(/up for testing/)
+    end
+
+    it "removes the global downfile" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/down", :reason => "down for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 503
+
+      delete "/force/down"
+      last_response.status.should == 200
+
+      get "/test/status"
+      last_response.should be_ok
+      last_response.body.should_not match(/down for testing/)
+    end
+
+    it "removes a service specific upfile" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      post "/force/up/test", :reason => "up for testing"
+      last_response.status.should == 201
+
+      get "/test/status"
+      last_response.status.should == 200
+      last_response.body.should match(/up for testing/)
+
+      delete "/force/up/test"
+      last_response.status.should == 200
+
+      get "/test/status"
+      last_response.status.should == 503
+    end
+
+    it "404s if there is no upfile" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+
+      delete "/up"
+
+      last_response.status.should == 404
+    end
+  end
+
+  describe "GET /:service/status" do
+    it "is successful when the service is passing" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      get "/test/status"
+
+      last_response.should be_ok
+      last_response.header["Content-Type"].should == "text/plain"
+      last_response.header["X-Health"].should == "100"
+      last_response.body.should match(/Health: 100/)
+      last_response.body.should match(/AlwaysAvailableDependency: OK/)
+      last_response.body.should include("ConstantMetric(100): 100")
+    end
+
+    it "is 'service unavailable' when the check fails" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      get "/test/status"
+
+      last_response.status.should == 503
+      last_response.header["Content-Type"].should == "text/plain"
+      last_response.header["X-Health"].should == "0"
+      last_response.body.should match(/Health: 0/)
+    end
+
+    it "is 'not found' when the service is unknown" do
+      get "/unknown/status"
+
+      last_response.status.should == 404
+      last_response.header["Content-Type"].should == "text/plain"
+    end
+
+    it "is 'service unavailable' when an up file and down file exists" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("up", "test").create("Up for testing")
+      LitmusPaper::StatusFile.new("down", "test").create("Down for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 503
+      last_response.body.should match(/Down for testing/)
+    end
+
+    it "is 'service available' when an up file exists" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("up", "test").create("Up for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 200
+      last_response.body.should match(/Up for testing/)
+    end
+
+    it "is 'service available' when an up file exists" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("up", "test").create("Up for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 200
+      last_response.body.should match(/Up for testing/)
+    end
+
+    it "is 'service unavailable' when a global down file and up file exists" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("global_down").create("Down for testing")
+      LitmusPaper::StatusFile.new("global_up").create("Up for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 503
+      last_response.body.should match(/Down for testing/)
+    end
+
+    it "is 'service unavailable' when a global down file exists" do
+      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("global_down").create("Down for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 503
+      last_response.body.should match(/Down for testing/)
+    end
+
+    it "is successful when a global up file exists" do
+      test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [ConstantMetric.new(100)])
+      LitmusPaper.services['test'] = test_service
+
+      LitmusPaper::StatusFile.new("global_up").create("Up for testing")
+
+      get "/test/status"
+
+      last_response.status.should == 200
+      last_response.body.should match(/Up for testing/)
+    end
+  end
+
+  describe "server errors" do
+    it "responds with a text/plain 500 response" do
+      old_environment = :test
+      begin
+        app.environment = :production
+        get "/test/error"
+        last_response.status.should == 500
+        last_response.headers["Content-Type"].should == "text/plain"
+        last_response.body.should == "Server Error"
+      ensure
+        app.environment = old_environment
+      end
+    end
+  end
+end

data/spec/litmus_paper/cli/admin_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+
+describe 'litmusctl' do
+  def _litmusctl(args)
+    `bundle exec ruby -I lib bin/litmusctl #{args} -p 9293`
+  end
+
+  before(:all) do
+    system "bundle exec ruby -I lib bin/litmus -p 9293 -d -D /tmp/litmus_paper -c #{TEST_CONFIG} -P /tmp/litmus.pid"
+  end
+
+  after(:all) do
+    system "kill -9 `cat /tmp/litmus.pid`"
+  end
+
+  describe 'list' do
+    it "returns the list of services running" do
+      _litmusctl('list').should match("test")
+    end
+  end
+
+  describe 'status' do
+    it 'returns the status of a service' do
+      _litmusctl('status test').should match("Health: 0")
+      _litmusctl('status passing_test').should match(/Health: \d\d/)
+    end
+
+    it "returns 'NOT FOUND' for a service that doesn't exist" do
+      _litmusctl('status unknown').should match('NOT FOUND')
+    end
+  end
+
+  describe "force" do
+    it "can create a global downfile" do
+      _litmusctl('force down -r "for testing"').should match("File created")
+
+      status = _litmusctl('status test')
+      status.should match(/Health: 0/)
+      status.should match(/for testing/)
+    end
+
+    it "creates a downfile" do
+      _litmusctl('force down test -r "for testing"').should match("File created")
+
+      status = _litmusctl('status test')
+      status.should match(/Health: 0/)
+      status.should match(/for testing/)
+    end
+
+    it 'removes an upfile for the service' do
+      _litmusctl('force up test -r "for testing"').should match("File created")
+      _litmusctl('force up test -d').should match("File deleted")
+
+      status = _litmusctl('status passing_test')
+      status.should match(/Health: \d\d/)
+      status.should_not match(/for testing/)
+    end
+
+    it "returns not found if downfile doesn't exist" do
+      _litmusctl('force down test -d').should match("NOT FOUND")
+    end
+  end
+end

data/spec/litmus_paper/cli/server_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'litmus_paper/cli/server'
+
+describe LitmusPaper::CLI::Server do
+  describe 'parse!' do
+    it 'parses litmus config file options' do
+      options = LitmusPaper::CLI::Server::Options.new.parse!(['-c', 'foo.conf'])
+      options[:litmus_config].should == 'foo.conf'
+    end
+
+    it 'parses the config dir options' do
+      options = LitmusPaper::CLI::Server::Options.new.parse!(['-D', '/tmp/foo'])
+      options[:config_dir].should == '/tmp/foo'
+    end
+  end
+end

data/spec/litmus_paper/configuration_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe LitmusPaper::Configuration do
+  describe "evaluate" do
+    it "configures a service" do
+      config = LitmusPaper::Configuration.new(TEST_CONFIG)
+      services = config.evaluate
+      services.has_key?('test').should == true
+    end
+  end
+
+  describe "include_files" do
+    it "configures a dir glob of services" do
+      config = LitmusPaper::Configuration.new(TEST_D_CONFIG)
+      services = config.evaluate
+      services.has_key?('test').should == true
+    end
+  end
+end

data/spec/litmus_paper/dependency/http_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+describe LitmusPaper::Dependency::HTTP do
+  before(:all) do
+    server_start = system "bundle exec rackup spec/support/http_test_server_config.ru --port 9294 --pid /tmp/http-test-server.pid --daemonize"
+    SpecHelper.wait_for_service :host => '127.0.0.1', :port => 9294
+    @url = "http://127.0.0.1:9294"
+  end
+
+  after(:all) do
+    system "kill -9 `cat /tmp/http-test-server.pid`"
+  end
+
+  describe "#available?" do
+    context "http method" do
+      it "uses the given http method when making the request" do
+        check = LitmusPaper::Dependency::HTTP.new("#{@url}/method", :method => "GET", :content => "POST")
+        check.should_not be_available
+
+        check = LitmusPaper::Dependency::HTTP.new("#{@url}/method", :method => "POST", :content => "POST")
+        check.should be_available
+      end
+    end
+
+    it "is true when response is 200" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/200")
+      check.should be_available
+    end
+
+    it "is true when response is 200 and expected content matches" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/200", :content => "200 OK")
+      check.should be_available
+    end
+
+    it "is false when response is 200, but does not match content" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/200", :content => "some text not in the response")
+      check.should_not be_available
+    end
+
+    it "is true when response is any 200 level response" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/201")
+      check.should be_available
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/202")
+      check.should be_available
+    end
+
+    it "is false when response is 500 " do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/500")
+      check.should_not be_available
+    end
+
+    it "is false when the response is 404" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/404")
+      check.should_not be_available
+    end
+
+    it "is false when the dependency is not available" do
+      check = LitmusPaper::Dependency::HTTP.new('http://127.0.0.1:7777')
+      check.should_not be_available
+    end
+  end
+
+  describe "to_s" do
+    it "is the name of the class and the url" do
+      check = LitmusPaper::Dependency::HTTP.new("#{@url}/status/500")
+      check.to_s.should == "Dependency::HTTP(http://127.0.0.1:9294/status/500)"
+    end
+  end
+end

data/spec/litmus_paper/dependency/tcp_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe LitmusPaper::Dependency::TCP do
+  before(:all) do
+    @server = TCPServer.new 3333
+  end
+
+  after(:all) do
+    @server.close
+  end
+
+  describe "#available?" do
+    it "is true when it's able to reach the ip and port" do
+      check = LitmusPaper::Dependency::TCP.new("127.0.0.1", 3333)
+      check.should be_available
+    end
+
+    it "is false when the ip is not available" do
+      check = LitmusPaper::Dependency::TCP.new("10.254.254.254", 3333)
+      check.should_not be_available
+    end
+
+    it "is false when the port is not available" do
+      check = LitmusPaper::Dependency::TCP.new("127.0.0.1", 3334)
+      check.should_not be_available
+    end
+  end
+
+  describe "to_s" do
+    it "is the name of the class and the ip and port" do
+      check = LitmusPaper::Dependency::TCP.new("127.0.0.1", 3333)
+      check.to_s.should == "Dependency::TCP(tcp://127.0.0.1:3333)"
+    end
+  end
+end

data/spec/litmus_paper/health_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe LitmusPaper::Health do
+  describe "ok?" do
+    it "is true when health is greater than 0" do
+      health = LitmusPaper::Health.new
+      health.perform(ConstantMetric.new(50))
+      health.should be_ok
+    end
+
+    it "is false when health is 0" do
+      health = LitmusPaper::Health.new
+      health.perform(ConstantMetric.new(0))
+      health.should_not be_ok
+    end
+  end
+
+  describe "perform" do
+    it "executes the check and adds its value to its health" do
+      health = LitmusPaper::Health.new
+      health.perform(ConstantMetric.new(50))
+      health.perform(ConstantMetric.new(25))
+      health.value.should == 75
+    end
+  end
+
+  describe "ensure" do
+    it "checks the dependency and modifies the value accordingly" do
+      health = LitmusPaper::Health.new
+      health.ensure(NeverAvailableDependency.new)
+      health.perform(ConstantMetric.new(50))
+      health.value.should == 0
+    end
+  end
+
+  describe "summary" do
+    it "includes the availablilty of dependencies" do
+      health = LitmusPaper::Health.new
+      health.ensure(NeverAvailableDependency.new)
+      health.ensure(AlwaysAvailableDependency.new)
+
+      health.summary.should match(/NeverAvailableDependency: FAIL/)
+      health.summary.should match(/AlwaysAvailableDependency: OK/)
+    end
+
+    it "includes the health of individual metrics" do
+      health = LitmusPaper::Health.new
+      health.perform(ConstantMetric.new(12))
+      health.perform(ConstantMetric.new(34))
+
+      health.summary.should include("ConstantMetric(12): 12")
+      health.summary.should include("ConstantMetric(34): 34")
+    end
+
+    it "only runs each metric once" do
+      health = LitmusPaper::Health.new
+      metric = ConstantMetric.new(12)
+      metric.should_receive(:current_health).once.and_return(12)
+
+      health.perform(metric)
+    end
+
+    it "only runs each dependency once" do
+      health = LitmusPaper::Health.new
+      dependency = AlwaysAvailableDependency.new
+      dependency.should_receive(:available?).once.and_return(true)
+
+      health.ensure(dependency)
+    end
+  end
+end

data/spec/litmus_paper/metric/available_memory_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe LitmusPaper::Metric::AvailableMemory do
+  describe "#current_health" do
+    it "multiplies weight by memory available" do
+      facter = StubFacter.new({"memorytotal" => "10 GB", "memoryfree" => "5 GB"})
+      memory = LitmusPaper::Metric::AvailableMemory.new(50, facter)
+      memory.current_health.should == 25
+    end
+
+    describe "#memory_total" do
+      it "is a positive integer" do
+        metric = LitmusPaper::Metric::AvailableMemory.new(50)
+        metric.memory_total.should > 1_000
+      end
+
+      it "is cached" do
+        Facter.should_receive(:value).once.and_return("10 MB")
+        metric = LitmusPaper::Metric::AvailableMemory.new(50)
+        metric.memory_total
+        metric.memory_total
+        metric.memory_total
+      end
+    end
+
+    describe "#memory_free" do
+      it "is a positive integer" do
+        metric = LitmusPaper::Metric::AvailableMemory.new(50)
+        metric.memory_free.should > 100
+      end
+    end
+
+    describe "#to_s" do
+      it "is the name of the check and the max weight" do
+        metric = LitmusPaper::Metric::AvailableMemory.new(50)
+        metric.to_s.should == "Metric::AvailableMemory(50)"
+      end
+    end
+  end
+end

data/spec/litmus_paper/metric/cpu_load_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe LitmusPaper::Metric::CPULoad do
+  describe "#current_health" do
+    it "is the percent of available cpu capacity" do
+      facter = StubFacter.new({"processorcount" => "4", "loadaverage" => "1.00 0.40 0.10"})
+      cpu_load = LitmusPaper::Metric::CPULoad.new(40, facter)
+      cpu_load.current_health.should == 30
+    end
+
+    it "is zero when the load is above one per core" do
+      facter = StubFacter.new({"processorcount" => "4", "loadaverage" => "20.00 11.40 10.00"})
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50, facter)
+      cpu_load.current_health.should == 0
+    end
+  end
+
+  describe "#processor_count" do
+    it "is a positive integer" do
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
+      cpu_load.processor_count.should > 0
+    end
+
+    it "is cached" do
+      Facter.should_receive(:value).once.and_return("10")
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
+      cpu_load.processor_count
+      cpu_load.processor_count
+      cpu_load.processor_count
+    end
+  end
+
+  describe "#load_average" do
+    it "is a floating point" do
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
+      cpu_load.load_average.should > 0.0
+    end
+  end
+
+  describe "#to_s" do
+    it "is the check name and the maximum weight" do
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
+      cpu_load.to_s.should == "Metric::CPULoad(50)"
+    end
+  end
+end