big_brother 0.4.1 → 0.5.0

data/lib/big_brother/app.rb

@@ -5,15 +5,13 @@ module BigBrother
     set :raise_errors, false
 
     get "/" do
-      running, stopped = BigBrother.clusters.values.partition(&:monitored?)
-
       [200, <<-CONTENT]
 Big Brother: #{BigBrother::VERSION}
 
 Running:
-#{running.map { |cluster| "+ #{cluster}\n" }.join}
+#{BigBrother.clusters.running.map { |cluster| "+ #{cluster}\n" }.join}
 Stopped:
-#{stopped.map { |cluster| "- #{cluster}\n" }.join}
+#{BigBrother.clusters.stopped.map { |cluster| "- #{cluster}\n" }.join}
       CONTENT
     end
 

data/lib/big_brother/cluster.rb

@@ -1,17 +1,37 @@
 module BigBrother
   class Cluster
-    attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name
+    attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name, :persistent, :ramp_up_time, :nagios
 
     def initialize(name, attributes = {})
       @name = name
-      @fwmark = attributes['fwmark']
-      @scheduler = attributes['scheduler']
-      @check_interval = attributes.fetch('check_interval', 1)
+      @fwmark = attributes[:fwmark]
+      @scheduler = attributes[:scheduler]
+      @persistent = attributes.fetch(:persistent, 300)
+      @check_interval = attributes.fetch(:check_interval, 1)
       @monitored = false
-      @nodes = attributes.fetch('nodes', [])
+      @nodes = attributes.fetch(:nodes, []).map { |node_config| _coerce_node(node_config) }
       @last_check = Time.new(0)
       @up_file = BigBrother::StatusFile.new('up', @name)
       @down_file = BigBrother::StatusFile.new('down', @name)
+      @ramp_up_time = attributes.fetch(:ramp_up_time, 60)
+      @has_downpage = attributes[:has_downpage]
+      @nagios = attributes[:nagios]
+    end
+
+    def _coerce_node(node_config)
+      node_config.is_a?(Node) ? node_config : Node.new(node_config)
+    end
+
+    def downpage_enabled?
+      @downpage_enabled
+    end
+
+    def find_node(address, port)
+      nodes.find{|node| node.address == address && node.port == port}
+    end
+
+    def has_downpage?
+      @has_downpage
     end
 
     def monitored?
@@ -20,7 +40,7 @@ module BigBrother
 
     def start_monitoring!
       BigBrother.logger.info "starting monitoring on cluster #{to_s}"
-      BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
+      BigBrother.ipvs.start_cluster(@fwmark, @scheduler, @persistent)
       @nodes.each do |node|
         BigBrother.ipvs.start_node(@fwmark, node.address, 100)
       end
@@ -62,12 +82,19 @@ module BigBrother
     def monitor_nodes
       @last_check = Time.now
       @nodes.each { |node| node.monitor(self) }
+
+      _check_downpage if has_downpage?
+      _notify_nagios if nagios
     end
 
     def to_s
       "#{@name} (#{@fwmark})"
     end
 
+    def ==(other)
+      fwmark == other.fwmark
+    end
+
     def up_file_exists?
       @up_file.exists?
     end
@@ -76,6 +103,13 @@ module BigBrother
       @down_file.exists?
     end
 
+    def incorporate_state(another_cluster)
+      nodes.each do |node|
+        node.incorporate_state(another_cluster.find_node(node.address, node.port))
+      end
+      self
+    end
+
     def _add_nodes(addresses)
       addresses.each do |address|
         BigBrother.logger.info "adding #{address} to cluster #{self}"
@@ -83,6 +117,39 @@ module BigBrother
       end
     end
 
+    def _add_maintenance_node
+      BigBrother.logger.info "adding 127.0.0.1 to cluster #{self}"
+      BigBrother.ipvs.start_node(fwmark, '127.0.0.1', 1)
+    end
+
+    def _check_downpage
+      total_health = @nodes.collect{ |n| n.weight || 0 }.reduce(:+)
+      if total_health <= 0
+        _add_maintenance_node unless downpage_enabled?
+        @downpage_enabled = true
+      else
+        _remove_maintenance_node if downpage_enabled?
+        @downpage_enabled = false
+      end
+    end
+
+    def _notify_nagios
+      nodes_down = @nodes.count{|n| n.weight == 0}
+      return if @last_node_count == nodes_down
+      if ((nodes_down / @nodes.count.to_f) >= 0.5)
+        BigBrother.nagios.send_critical(nagios[:host], nagios[:check], "50% of nodes are down", nagios[:server])
+      elsif nodes_down > 0
+        BigBrother.nagios.send_warning(nagios[:host], nagios[:check], "a node is down", nagios[:server])
+      else
+        BigBrother.nagios.send_ok(nagios[:host], nagios[:check], "all nodes up", nagios[:server])
+      end
+      @last_node_count = nodes_down
+    end
+
+    def _remove_maintenance_node
+      BigBrother.ipvs.stop_node(fwmark, '127.0.0.1')
+    end
+
     def _remove_nodes(addresses)
       addresses.each do |address|
         BigBrother.logger.info "removing #{address} to cluster #{self}"

data/lib/big_brother/cluster_collection.rb

@@ -0,0 +1,34 @@
+require 'forwardable'
+
+module BigBrother
+  class ClusterCollection
+    extend Forwardable
+    def_delegators :@clusters, :[], :[]=, :size, :clear
+
+    def initialize
+      @clusters = {}
+    end
+
+    def config(new_clusters)
+      new_clusters.each do |cluster_name, cluster|
+        if @clusters.key?(cluster_name)
+          @clusters[cluster_name] = cluster.incorporate_state(@clusters[cluster_name])
+        else
+          @clusters[cluster_name] = cluster
+        end
+      end
+    end
+
+    def running
+      @clusters.values.select(&:monitored?)
+    end
+
+    def stopped
+      @clusters.values.reject(&:monitored?)
+    end
+
+    def ready_for_check
+      @clusters.values.select(&:needs_check?)
+    end
+  end
+end

data/lib/big_brother/configuration.rb

@@ -1,18 +1,34 @@
 module BigBrother
   class Configuration
-    def self.evaluate(file)
-      yaml = YAML.load(File.read(file))
-      assoc_array = yaml.map do |name, values|
-        nodes = _parse_nodes(values.delete('nodes'))
-        [name, Cluster.new(name, values.merge('nodes' => nodes))]
+    GLOBAL_CONFIG_KEY = '_big_brother'
+
+    def self.from_file(config_file)
+      config = YAML.load_file(config_file)
+      defaults = config.delete(GLOBAL_CONFIG_KEY)
+
+      config.inject({}) do |clusters, (cluster_name, cluster_values)|
+        cluster_details = _apply_defaults(defaults, cluster_values)
+        clusters.merge(cluster_name => Cluster.new(cluster_name, _deeply_symbolize_keys(cluster_details)))
       end
+    end
 
-      Hash[assoc_array]
+    def self._deeply_symbolize_keys(value)
+      if value.is_a?(Hash)
+        value.inject({}) do |symbolized_hash, (hash_key, hash_value)|
+          symbolized_hash[hash_key.to_sym] = _deeply_symbolize_keys(hash_value)
+          symbolized_hash
+        end
+      elsif value.is_a?(Array)
+        value.map { |item| _deeply_symbolize_keys(item) }
+      else
+        value
+      end
     end
 
-    def self._parse_nodes(nodes)
-      nodes.map do |values|
-        Node.new(values['address'], values['port'], values['path'])
+    def self._apply_defaults(defaults_hash, settings_hash)
+      return settings_hash unless defaults_hash
+      defaults_hash.merge(settings_hash) do |key, oldval, newval|
+        oldval.is_a?(Hash) && newval.is_a?(Hash) ? _apply_defaults(oldval, newval) : newval
       end
     end
   end

data/lib/big_brother/ipvs.rb

@@ -4,8 +4,8 @@ module BigBrother
       @executor = executor
     end
 
-    def start_cluster(fwmark, scheduler)
-      @executor.invoke("ipvsadm --add-service --fwmark-service #{fwmark} --scheduler #{scheduler}")
+    def start_cluster(fwmark, scheduler, persistent)
+      @executor.invoke("ipvsadm --add-service --fwmark-service #{fwmark} --scheduler #{scheduler} --persistent #{persistent}")
     end
 
     def stop_cluster(fwmark)

data/lib/big_brother/nagios.rb

@@ -0,0 +1,30 @@
+module BigBrother
+  class Nagios
+    module Code
+      Ok       = 0
+      Warning  = 1
+      Critical = 2
+      Unknown  = 3
+    end
+
+    def initialize(executor = ShellExecutor.new)
+      @executor = executor
+    end
+
+    def send_critical(host, check, message, server)
+      _send_passive(host, check, Code::Critical, "CRITICAL #{message}", server)
+    end
+
+    def send_ok(host, check, message, server)
+      _send_passive(host, check, Code::Ok, "OK #{message}", server)
+    end
+
+    def send_warning(host, check, message, server)
+      _send_passive(host, check, Code::Warning, "WARNING #{message}", server)
+    end
+
+    def _send_passive(host, check, code, message, server)
+      @executor.invoke("echo '#{host},#{check},#{code},#{message}' | send_nsca -H #{server} -d ,")
+    end
+  end
+end

data/lib/big_brother/node.rb

@@ -2,13 +2,25 @@ require 'net/http'
 
 module BigBrother
   class Node
-    attr_reader :address, :port, :path
+    attr_reader :address, :port, :path, :start_time, :weight
 
-    def initialize(address, port, path)
-      @address = address
-      @port = port
-      @path = path
-      @weight = nil
+    def initialize(attributes={})
+      @address = attributes[:address]
+      @port = attributes[:port]
+      @path = attributes[:path]
+      @weight = attributes[:weight]
+      @start_time = attributes.fetch(:start_time, Time.now.to_i)
+    end
+
+    def age
+      Time.now.to_i - @start_time
+    end
+
+    def incorporate_state(another_node)
+      if another_node
+        @weight = another_node.weight
+        @start_time = another_node.start_time
+      end
     end
 
     def invalidate_weight!
@@ -24,13 +36,26 @@ module BigBrother
       end
     end
 
+    def ==(other)
+      address == other.address && port == other.port
+    end
+
     def _determine_weight(cluster)
       if cluster.up_file_exists?
         100
       elsif cluster.down_file_exists?
         0
       else
-        BigBrother::HealthFetcher.current_health(@address, @port, @path)
+        _weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time)
+      end
+    end
+
+    def _weight_health(health, ramp_up_time)
+      current_age = age
+      if current_age < ramp_up_time
+        (health * (current_age / ramp_up_time.to_f)).to_i
+      else
+        health
       end
     end
   end

data/lib/big_brother/status_file.rb

@@ -1,7 +1,7 @@
 module BigBrother
   class StatusFile
     def initialize(*filenames)
-      @path = File.join(BigBrother.config_dir, *filenames)
+      @path = File.join(BigBrother.config_dir, *[filenames.map(&:to_s)])
     end
 
     def content

data/lib/big_brother/ticker.rb

@@ -19,7 +19,7 @@ module BigBrother
 
     def self.tick
       @outstanding_ticks += 1
-      BigBrother.clusters.values.select(&:needs_check?).each do |cluster|
+      BigBrother.clusters.ready_for_check.each do |cluster|
         BigBrother.logger.debug("Monitoring cluster #{cluster.name}")
         cluster.monitor_nodes
       end

data/lib/big_brother/version.rb

@@ -1,3 +1,3 @@
 module BigBrother
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/big_brother.rb

@@ -9,10 +9,12 @@ require 'sinatra/synchrony'
 
 require 'big_brother/app'
 require 'big_brother/cluster'
+require 'big_brother/cluster_collection'
 require 'big_brother/configuration'
 require 'big_brother/health_fetcher'
 require 'big_brother/ipvs'
 require 'big_brother/logger'
+require 'big_brother/nagios'
 require 'big_brother/node'
 require 'big_brother/shell_executor'
 require 'big_brother/status_file'
@@ -25,16 +27,17 @@ require 'thin/callback_rack_handler'
 
 module BigBrother
   class << self
-    attr_accessor :ipvs, :clusters, :config_dir, :logger
+    attr_accessor :ipvs, :nagios, :clusters, :config_dir, :logger
   end
 
   self.ipvs = IPVS.new
-  self.clusters = {}
+  self.nagios = Nagios.new
+  self.clusters = BigBrother::ClusterCollection.new
   self.logger = BigBrother::Logger.new
 
   def self.configure(filename)
     @config_file = filename
-    @clusters = BigBrother::Configuration.evaluate(filename)
+    @clusters.config(BigBrother::Configuration.from_file(filename))
   end
 
   def self.start_ticker!

data/spec/big_brother/app_spec.rb

@@ -106,7 +106,7 @@ module BigBrother
         last_response.status.should == 200
         last_response.body.should == "OK"
         BigBrother.clusters['test'].should be_monitored
-        @stub_executor.commands.should include("ipvsadm --add-service --fwmark-service 100 --scheduler wrr")
+        @stub_executor.commands.should include("ipvsadm --add-service --fwmark-service 100 --scheduler wrr --persistent 300")
         @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
         @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 100")
       end

data/spec/big_brother/cluster_collection_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe BigBrother::ClusterCollection do
+  describe "config" do
+    it "adds the provided clusters into its collection" do
+      clusters_from_config = {
+        'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
+        'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),
+        'test3' => Factory.cluster(:name => 'test3', :fwmark => 103)
+      }
+      collection = BigBrother::ClusterCollection.new
+
+      collection.config(clusters_from_config)
+
+      collection['test1'].should == clusters_from_config['test1']
+      collection['test2'].should == clusters_from_config['test2']
+      collection['test3'].should == clusters_from_config['test3']
+    end
+
+    it "incorporates the state of clusters that exist" do
+      collection = BigBrother::ClusterCollection.new
+
+      collection['existing_cluster'] = Factory.cluster(:name => 'existing_cluster')
+
+      cluster_from_config = Factory.cluster(:name => 'existing_cluster')
+      cluster_from_config.should_receive(:incorporate_state).with(collection['existing_cluster'])
+
+      collection.config({'existing_cluster' => cluster_from_config})
+    end
+  end
+
+  describe "running" do
+    it "returns the clusters in the collection that are currently running" do
+      clusters_from_config = {
+        'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
+        'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),
+        'test3' => Factory.cluster(:name => 'test3', :fwmark => 103)
+      }
+      collection = BigBrother::ClusterCollection.new
+
+      collection.config(clusters_from_config)
+      clusters_from_config['test1'].start_monitoring!
+      clusters_from_config['test2'].start_monitoring!
+
+      collection.running.should == [clusters_from_config['test1'], clusters_from_config['test2']]
+    end
+  end
+
+  describe "stopped" do
+    it "returns the clusters in the collection that are not running" do
+      clusters_from_config = {
+        'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
+        'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),
+        'test3' => Factory.cluster(:name => 'test3', :fwmark => 103)
+      }
+      collection = BigBrother::ClusterCollection.new
+
+      collection.config(clusters_from_config)
+      clusters_from_config['test1'].start_monitoring!
+      clusters_from_config['test2'].start_monitoring!
+
+      collection.stopped.should == [clusters_from_config['test3']]
+    end
+  end
+
+  describe "ready_for_check" do
+    it "returns the clusters in the collection that need checking" do
+      clusters_from_config = {
+        'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
+        'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),
+        'test3' => Factory.cluster(:name => 'test3', :fwmark => 103)
+      }
+      collection = BigBrother::ClusterCollection.new
+
+      collection.config(clusters_from_config)
+      clusters_from_config['test1'].stub(:needs_check?).and_return(true)
+      clusters_from_config['test2'].stub(:needs_check?).and_return(true)
+      clusters_from_config['test3'].stub(:needs_check?).and_return(false)
+
+      collection.ready_for_check.should == [clusters_from_config['test1'], clusters_from_config['test2']]
+    end
+  end
+end

data/spec/big_brother/cluster_spec.rb

@@ -13,7 +13,7 @@ describe BigBrother::Cluster do
       cluster = Factory.cluster(:fwmark => 100, :scheduler => 'wrr')
 
       cluster.start_monitoring!
-      @stub_executor.commands.should include('ipvsadm --add-service --fwmark-service 100 --scheduler wrr')
+      @stub_executor.commands.should include('ipvsadm --add-service --fwmark-service 100 --scheduler wrr --persistent 300')
     end
   end
 
@@ -77,6 +77,124 @@ describe BigBrother::Cluster do
 
       cluster.monitor_nodes
     end
+
+    it "enables a downpage if none of the nodes have health > 0" do
+      node1 = Factory.node
+      node2 = Factory.node
+      cluster = Factory.cluster(:has_downpage => true, :nodes => [node1, node2])
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(0)
+
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+      cluster.downpage_enabled?.should be_true
+    end
+
+    it "does not enable a downpage if the cluster does not have a downpage enabled" do
+      node1 = Factory.node
+      node2 = Factory.node
+      cluster = Factory.cluster(:has_downpage => false, :nodes => [node1, node2])
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(0)
+
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+      cluster.downpage_enabled?.should be_false
+    end
+
+    it "adds a downpage node to IPVS when down" do
+      node1 = Factory.node
+      node2 = Factory.node
+      cluster = Factory.cluster(:has_downpage => true, :nodes => [node1, node2], :fwmark => 1)
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(0)
+
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+
+      @stub_executor.commands.last.should == "ipvsadm --add-server --fwmark-service 1 --real-server 127.0.0.1 --ipip --weight 1"
+    end
+
+    it "removes downpage node from IPVS if it exists and cluster is up" do
+      node1 = Factory.node
+      node2 = Factory.node
+      cluster = Factory.cluster(:has_downpage => true, :nodes => [node1, node2], :fwmark => 1)
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(0)
+
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(10)
+      cluster.monitor_nodes
+
+      @stub_executor.commands.last.should == "ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.0.1"
+    end
+
+    context "nagios" do
+      it "sends critical if at least half of all nodes are down" do
+        node1 = Factory.node(:address => '192.168.0.1')
+        node2 = Factory.node(:address => '192.168.0.2')
+        cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
+
+        node1.stub(:_determine_weight).and_return(0)
+        node2.stub(:_determine_weight).and_return(10)
+
+        cluster.start_monitoring!
+        cluster.monitor_nodes
+        @stub_executor.commands.should include("echo 'prod.load,test1_check,2,CRITICAL 50% of nodes are down' | send_nsca -H server.foo -d ,")
+      end
+
+      it "does not resend a Nagios check if the state does not change" do
+        node1 = Factory.node(:address => '192.168.0.1')
+        node2 = Factory.node(:address => '192.168.0.2')
+        cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
+
+        node1.stub(:_determine_weight).and_return(0)
+        node2.stub(:_determine_weight).and_return(10)
+
+        cluster.start_monitoring!
+        cluster.monitor_nodes
+        @stub_executor.clear_commands!
+
+        cluster.monitor_nodes
+        @stub_executor.commands.should_not include("echo 'prod.load,test1_check,2,CRITICAL 50% of nodes are down' | send_nsca -H server.foo -d ,")
+      end
+
+      it "sends info if at least half of one node is down" do
+        node1 = Factory.node(:address => '192.168.0.1')
+        node2 = Factory.node(:address => '192.168.0.2')
+        node3 = Factory.node(:address => '192.168.0.3')
+        cluster = Factory.cluster(:nodes => [node1, node2, node3], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
+
+        node1.stub(:_determine_weight).and_return(0)
+        node2.stub(:_determine_weight).and_return(10)
+        node3.stub(:_determine_weight).and_return(10)
+
+        cluster.start_monitoring!
+        cluster.monitor_nodes
+        @stub_executor.commands.should include("echo 'prod.load,test1_check,1,WARNING a node is down' | send_nsca -H server.foo -d ,")
+      end
+
+      it "sends ok if all nodes back up" do
+        node1 = Factory.node(:address => '192.168.0.1')
+        node2 = Factory.node(:address => '192.168.0.2')
+        cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
+        node1.stub(:_determine_weight).and_return(0)
+        node2.stub(:_determine_weight).and_return(10)
+
+        cluster.start_monitoring!
+        cluster.monitor_nodes
+        @stub_executor.commands.should include("echo 'prod.load,test1_check,2,CRITICAL 50% of nodes are down' | send_nsca -H server.foo -d ,")
+        @stub_executor.clear_commands!
+
+        node1.stub(:_determine_weight).and_return(10)
+        node2.stub(:_determine_weight).and_return(10)
+        cluster.monitor_nodes
+        @stub_executor.commands.should include("echo 'prod.load,test1_check,0,OK all nodes up' | send_nsca -H server.foo -d ,")
+      end
+    end
+
   end
 
   describe "#resume_monitoring!" do
@@ -143,6 +261,18 @@ describe BigBrother::Cluster do
     end
   end
 
+  describe "#==" do
+    it "is true if two clusters have the same fwmark" do
+      cluster1 = Factory.cluster(:fwmark => '100')
+      cluster2 = Factory.cluster(:fwmark => '200')
+
+      cluster1.should_not == cluster2
+
+      cluster2 = Factory.cluster(:fwmark => '100')
+      cluster1.should == cluster2
+    end
+  end
+
   describe "#up_file_exists?" do
     it "returns true when an up file exists" do
       cluster = Factory.cluster(:name => 'name')
@@ -164,4 +294,23 @@ describe BigBrother::Cluster do
       cluster.down_file_exists?.should be_true
     end
   end
+
+  describe "incorporate_state" do
+    it "finds any equivalent nodes from the provided cluster, incorporates their state, and returns self" do
+      original_node1 = Factory.node(:address => '127.0.0.1')
+      original_node2 = Factory.node(:address => '127.0.1.1')
+      original_cluster = Factory.cluster(:nodes => [original_node1, original_node2])
+
+      config_node1 = Factory.node(:address => '127.0.0.1')
+      config_node2 = Factory.node(:address => '127.0.1.1')
+      config_cluster = Factory.cluster(:nodes => [config_node1, config_node2])
+
+      config_node1.should_receive(:incorporate_state).with(original_node1)
+      config_node2.should_receive(:incorporate_state).with(original_node2)
+
+      retval = config_cluster.incorporate_state(original_cluster)
+
+      retval.should == config_cluster
+    end
+  end
 end

data/spec/big_brother/configuration_spec.rb

@@ -1,17 +1,24 @@
 require 'spec_helper'
 
 describe BigBrother::Configuration do
-  describe '.evaluate' do
-    it 'returns a hash of clusters' do
-      clusters = BigBrother::Configuration.evaluate(TEST_CONFIG)
+  describe 'self.from_file' do
+    it 'maintain a collection of clusters' do
+      clusters = BigBrother::Configuration.from_file(TEST_CONFIG)
 
       clusters['test1'].check_interval.should == 1
       clusters['test1'].scheduler.should == 'wrr'
       clusters['test1'].fwmark.should == 1
+      clusters['test1'].persistent.should == 20
+      clusters['test1'].ramp_up_time.should == 120
+      clusters['test1'].has_downpage?.should == true
+      clusters['test1'].nagios[:check].should == 'test1_status'
+      clusters['test1'].nagios[:host].should == 'prod-load'
+      clusters['test1'].nagios[:server].should == 'nsca.host'
 
-      clusters['test2'].check_interval.should == 1
+      clusters['test2'].check_interval.should == 2
       clusters['test2'].scheduler.should == 'wrr'
       clusters['test2'].fwmark.should == 2
+      clusters['test2'].ramp_up_time.should == 60
 
       clusters['test3'].check_interval.should == 1
       clusters['test3'].scheduler.should == 'wrr'
@@ -19,7 +26,7 @@ describe BigBrother::Configuration do
     end
 
     it 'populates a clusters nodes' do
-      clusters = BigBrother::Configuration.evaluate(TEST_CONFIG)
+      clusters = BigBrother::Configuration.from_file(TEST_CONFIG)
 
       clusters['test1'].nodes.length.should == 2
 
@@ -31,5 +38,64 @@ describe BigBrother::Configuration do
       clusters['test1'].nodes[1].port == '9002'
       clusters['test1'].nodes[1].path == '/test/valid'
     end
+
+    it 'allows a default cluster configuration under the global config key' do
+      config_file = Tempfile.new('config.yml')
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF.gsub(/^ {10}/,''))
+          ---
+          _big_brother:
+            check_interval: 2
+            scheduler: wrr
+            nagios:
+              server: 127.0.0.2
+              host: ha-services
+          test_without_overrides:
+            fwmark: 2
+            nagios:
+              check: test_check
+            nodes:
+            - address: 127.0.0.1
+              port: 9001
+              path: /test/invalid
+          test_with_overrides:
+            fwmark: 3
+            scheduler: wlc
+            nagios:
+              host: override-host
+              check: test_overrides_check
+            nodes:
+            - address: 127.0.0.1
+              port: 9001
+              path: /test/invalid
+        EOF
+      end
+
+      clusters = BigBrother::Configuration.from_file(config_file)
+
+      clusters['test_without_overrides'].check_interval.should == 2
+      clusters['test_without_overrides'].scheduler.should == 'wrr'
+      clusters['test_without_overrides'].fwmark.should == 2
+      clusters['test_without_overrides'].nagios[:server].should == '127.0.0.2'
+      clusters['test_without_overrides'].nagios[:host].should == 'ha-services'
+      clusters['test_without_overrides'].nagios[:check].should == 'test_check'
+
+
+      clusters['test_with_overrides'].check_interval.should == 2
+      clusters['test_with_overrides'].scheduler.should == 'wlc'
+      clusters['test_with_overrides'].fwmark.should == 3
+      clusters['test_with_overrides'].nagios[:server].should == '127.0.0.2'
+      clusters['test_with_overrides'].nagios[:host].should == 'override-host'
+      clusters['test_with_overrides'].nagios[:check].should == 'test_overrides_check'
+    end
+  end
+
+  describe '_apply_defaults' do
+    it 'returns a new hash with the defaults hash and the settings hash merged recursively' do
+      defaults = {:foo => {:bar => 1}}
+      settings = {:foo => {:baz => 2}}
+      h = BigBrother::Configuration._apply_defaults(defaults, settings)
+      h.should == {:foo => {:bar => 1, :baz => 2}}
+    end
   end
 end

data/spec/big_brother/node_spec.rb

@@ -15,6 +15,27 @@ describe BigBrother::Node do
       @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
     end
 
+    it "a node's health should increase linearly over the specified ramp up time" do
+      BigBrother::HealthFetcher.stub(:current_health).and_return(100)
+      Time.stub(:now).and_return(1345043600)
+
+      node = Factory.node(:address => '127.0.0.1')
+      cluster = Factory.cluster(:ramp_up_time => 60, :fwmark => 100, :nodes => [node])
+      cluster.start_monitoring!
+
+      Time.stub(:now).and_return(1345043630)
+      node.monitor(cluster)
+      @stub_executor.commands.last.should == "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 50"
+
+      Time.stub(:now).and_return(1345043645)
+      node.monitor(cluster)
+      @stub_executor.commands.last.should == "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 75"
+
+      Time.stub(:now).and_return(1345043720)
+      node.monitor(cluster)
+      @stub_executor.commands.last.should == "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100"
+    end
+
     it "sets the weight to 100 for each node if an up file exists" do
       BigBrother::HealthFetcher.stub(:current_health).and_return(56)
       node = Factory.node(:address => '127.0.0.1')
@@ -86,4 +107,41 @@ describe BigBrother::Node do
       @stub_executor.commands.should == []
     end
   end
+
+  describe "#==" do
+    it "is true when two nodes have the same address and port" do
+      node1 = Factory.node(:address => "127.0.0.1", :port => "8000")
+      node2 = Factory.node(:address => "127.0.0.1", :port => "8001")
+      node1.should_not == node2
+
+      node2 = Factory.node(:address => "127.0.0.2", :port => "8000")
+      node1.should_not == node2
+
+      node2 = Factory.node(:address => "127.0.0.1", :port => "8000")
+      node1.should == node2
+    end
+  end
+
+  describe "age" do
+    it "is the time in seconds since the node started" do
+      Time.stub(:now).and_return(1345043612)
+      node = Factory.node(:address => "127.0.0.1")
+
+      node.age.should == 0
+    end
+  end
+
+  describe "incorporate_state" do
+    it "takes the weight and the start time from the other node, but leaves rest of config" do
+      original_start_time = Time.now
+      node_with_state = Factory.node(:path => '/old/path', :start_time => original_start_time, :weight => 65)
+      node_from_config = Factory.node(:path => '/new/path', :start_time => Time.now, :weight => 100)
+
+      node_from_config.incorporate_state(node_with_state)
+
+      node_from_config.path.should == '/new/path'
+      node_from_config.start_time.should == original_start_time
+      node_from_config.weight.should == 65
+    end
+  end
 end

data/spec/big_brother/status_file_spec.rb

@@ -1,6 +1,15 @@
 require 'spec_helper'
 
 describe BigBrother::StatusFile do
+  describe "initialize" do
+    it "accepts symbols as arguments, since that's how they will come from configuration" do
+      status_file = BigBrother::StatusFile.new(:foo)
+      status_file.create("for testing")
+
+      status_file.exists?.should == true
+    end
+  end
+
   describe "create" do
     it "creates a nested file" do
       status_file = BigBrother::StatusFile.new("foo", "bar")

data/spec/big_brother_spec.rb

@@ -21,13 +21,14 @@ HTTP
       spec.run
       server.stop
     end
+
     it "reconfigures the clusters" do
       config_file = Tempfile.new('config.yml')
       File.open(config_file, 'w') do |f|
         f.puts(<<-EOF)
 ---
 test1:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 1
   nodes:
@@ -44,7 +45,7 @@ EOF
         f.puts(<<-EOF)
 ---
 test1:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 1
   nodes:
@@ -57,15 +58,62 @@ EOF
       BigBrother.clusters['test1'].nodes.first.path.should == "/test/another/path"
     end
 
+    it "maintains the start_time and weight of existing nodes after reconfiguring" do
+      Time.stub(:now).and_return(Time.at(1345043600))
+      config_file = Tempfile.new('config.yml')
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  check_interval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/valid
+EOF
+      end
+      BigBrother.configure(config_file)
+      BigBrother.start_ticker!
+
+      Time.stub(:now).and_return(Time.at(1345043700))
+      start_time = BigBrother.clusters['test1'].nodes[0].start_time
+      weight = BigBrother.clusters['test1'].nodes[0].weight
+
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  check_interval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/valid
+  - address: 127.0.0.2
+    port: 9001
+    path: /test/valid
+EOF
+      end
+      BigBrother.reconfigure
+      BigBrother.clusters['test1'].nodes[0].start_time.should == start_time
+      BigBrother.clusters['test1'].nodes[0].weight.should == weight
+      BigBrother.clusters['test1'].nodes[1].start_time.should == 1345043700
+      BigBrother.clusters['test1'].nodes[1].weight.should be_nil
+    end
+
     it "stops the ticker and reconfigures after it has finished all its ticks" do
       config_file = Tempfile.new('config.yml')
       File.open(config_file, 'w') do |f|
         f.puts(<<-EOF)
 ---
 test1:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 1
+  ramp_up_time: 0
   nodes:
   - address: 127.0.0.1
     port: 9001
@@ -84,7 +132,7 @@ EOF
         f.puts(<<-EOF)
 ---
 test1:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 1
   nodes:

data/spec/spec_helper.rb

@@ -15,12 +15,13 @@ RSpec.configure do |config|
     ipvs = BigBrother.ipvs
     @stub_executor = StubExecutor.new
     BigBrother.ipvs = BigBrother::IPVS.new(@stub_executor)
+    BigBrother.nagios = BigBrother::Nagios.new(@stub_executor)
     spec.run
     BigBrother.ipvs = ipvs
   end
 
   config.before(:each) do
-    BigBrother.clusters.replace({})
+    BigBrother.clusters.clear
     FileUtils.rm_rf(BigBrother.config_dir)
     BigBrother.logger = NullLogger.new
   end

data/spec/support/example_config.yml

@@ -1,8 +1,15 @@
 ---
 test1:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 1
+  persistent: 20
+  ramp_up_time: 120
+  has_downpage: true
+  nagios:
+    server: nsca.host
+    check: test1_status
+    host: prod-load
   nodes:
   - address: 127.0.0.1
     port: 9001
@@ -11,7 +18,7 @@ test1:
     port: 9002
     path: /test/valid
 test2:
-  checkInterval: 1
+  check_interval: 2
   scheduler: wrr
   fwmark: 2
   nodes:
@@ -22,7 +29,7 @@ test2:
     port: 9002
     path: /test/invalid
 test3:
-  checkInterval: 1
+  check_interval: 1
   scheduler: wrr
   fwmark: 3
   nodes:

data/spec/support/factories/cluster_factory.rb

@@ -3,11 +3,12 @@ class Factory
     BigBrother::Cluster.new(
       overrides.fetch(:name, 'test'),
       {
-        'fwmark' => overrides.fetch(:fwmark, 100),
-        'scheduler' => overrides.fetch(:scheduler, 'wrr'),
-        'check_interval' => overrides.fetch(:check_interval, 1),
-        'nodes' => overrides.fetch(:nodes, [Factory.node])
-      }
+        :fwmark         => 100,
+        :scheduler      => 'wrr',
+        :check_interval => 1,
+        :nodes          => [Factory.node],
+        :ramp_up_time   => 0
+      }.merge(overrides)
     )
   end
 end

data/spec/support/factories/node_factory.rb

@@ -1,9 +1,11 @@
 class Factory
   def self.node(overrides = {})
     BigBrother::Node.new(
-      overrides.fetch(:address, 'localhost'),
-      overrides.fetch(:port, 8081),
-      overrides.fetch(:path, '/test/status')
+      {
+        :address => 'localhost',
+        :port    => 8081,
+        :path    => '/test/status'
+      }.merge(overrides)
     )
   end
 end

data/spec/support/stub_executor.rb

@@ -15,4 +15,8 @@ class StubExecutor
     @responses[command] ||= []
     @responses[command].push [output, status]
   end
+
+  def clear_commands!
+    @commands = []
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: big_brother
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-03 00:00:00.000000000 Z
+date: 2012-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70295616819500 !ruby/object:Gem::Requirement
+  requirement: &70176230223860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.3.1
   type: :runtime
   prerelease: false
-  version_requirements: *70295616819500
+  version_requirements: *70176230223860
 - !ruby/object:Gem::Dependency
   name: async-rack
-  requirement: &70295616818500 !ruby/object:Gem::Requirement
+  requirement: &70176230223060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70295616818500
+  version_requirements: *70176230223060
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70295616817840 !ruby/object:Gem::Requirement
+  requirement: &70176230222360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70295616817840
+  version_requirements: *70176230222360
 - !ruby/object:Gem::Dependency
   name: rack-fiber_pool
-  requirement: &70295616817080 !ruby/object:Gem::Requirement
+  requirement: &70176230221420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '0.9'
   type: :runtime
   prerelease: false
-  version_requirements: *70295616817080
+  version_requirements: *70176230221420
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70295616816140 !ruby/object:Gem::Requirement
+  requirement: &70176230220360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -68,10 +68,10 @@ dependencies:
         version: 1.0.0.beta.100
   type: :runtime
   prerelease: false
-  version_requirements: *70295616816140
+  version_requirements: *70176230220360
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70295616814920 !ruby/object:Gem::Requirement
+  requirement: &70176230219060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -79,10 +79,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70295616814920
+  version_requirements: *70176230219060
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &70295616814160 !ruby/object:Gem::Requirement
+  requirement: &70176230303940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -90,10 +90,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70295616814160
+  version_requirements: *70176230303940
 - !ruby/object:Gem::Dependency
   name: em-resolv-replace
-  requirement: &70295616813240 !ruby/object:Gem::Requirement
+  requirement: &70176230302240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -101,10 +101,10 @@ dependencies:
         version: '1.1'
   type: :runtime
   prerelease: false
-  version_requirements: *70295616813240
+  version_requirements: *70176230302240
 - !ruby/object:Gem::Dependency
   name: em-syslog
-  requirement: &70295616811960 !ruby/object:Gem::Requirement
+  requirement: &70176230301680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -112,10 +112,10 @@ dependencies:
         version: 0.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70295616811960
+  version_requirements: *70176230301680
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70295616839480 !ruby/object:Gem::Requirement
+  requirement: &70176230300780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -123,10 +123,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *70295616839480
+  version_requirements: *70176230300780
 - !ruby/object:Gem::Dependency
   name: rack-test
-  requirement: &70295616838880 !ruby/object:Gem::Requirement
+  requirement: &70176230300280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -134,10 +134,10 @@ dependencies:
         version: 0.6.1
   type: :development
   prerelease: false
-  version_requirements: *70295616838880
+  version_requirements: *70176230300280
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70295616838460 !ruby/object:Gem::Requirement
+  requirement: &70176230299880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -145,10 +145,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70295616838460
+  version_requirements: *70176230299880
 - !ruby/object:Gem::Dependency
   name: rake_commit
-  requirement: &70295616837740 !ruby/object:Gem::Requirement
+  requirement: &70176230299300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -156,10 +156,10 @@ dependencies:
         version: '0.13'
   type: :development
   prerelease: false
-  version_requirements: *70295616837740
+  version_requirements: *70176230299300
 - !ruby/object:Gem::Dependency
   name: vagrant
-  requirement: &70295616837060 !ruby/object:Gem::Requirement
+  requirement: &70176230298860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -167,7 +167,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70295616837060
+  version_requirements: *70176230298860
 description: IPVS backend supervisor
 email:
 - code@getbraintree.com
@@ -191,10 +191,12 @@ files:
 - lib/big_brother/app.rb
 - lib/big_brother/cli.rb
 - lib/big_brother/cluster.rb
+- lib/big_brother/cluster_collection.rb
 - lib/big_brother/configuration.rb
 - lib/big_brother/health_fetcher.rb
 - lib/big_brother/ipvs.rb
 - lib/big_brother/logger.rb
+- lib/big_brother/nagios.rb
 - lib/big_brother/node.rb
 - lib/big_brother/shell_executor.rb
 - lib/big_brother/status_file.rb
@@ -205,6 +207,7 @@ files:
 - lib/thin/callback_rack_handler.rb
 - lib/thin/callbacks.rb
 - spec/big_brother/app_spec.rb
+- spec/big_brother/cluster_collection_spec.rb
 - spec/big_brother/cluster_spec.rb
 - spec/big_brother/configuration_spec.rb
 - spec/big_brother/health_fetcher_spec.rb
@@ -245,12 +248,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.12
 signing_key: 
 specification_version: 3
 summary: Process to monitor and update weights for servers in an IPVS pool
 test_files:
 - spec/big_brother/app_spec.rb
+- spec/big_brother/cluster_collection_spec.rb
 - spec/big_brother/cluster_spec.rb
 - spec/big_brother/configuration_spec.rb
 - spec/big_brother/health_fetcher_spec.rb