big_brother 0.6.8 → 0.8.7

data/spec/big_brother/active_passive_cluster_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+
+describe BigBrother::ActivePassiveCluster do
+  describe "#start_monitoring!" do
+    it "starts only the node with the least priority in IPVS" do
+      cluster = Factory.active_passive_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:priority => 0, :address => "127.0.0.1"),
+          Factory.node(:priority => 1, :address => "127.0.0.2"),
+        ],
+      )
+
+      cluster.start_monitoring!
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1')
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 1')
+    end
+
+    it "monitors a node before adding it to ipvs" do
+      cluster = Factory.active_passive_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:priority => 0, :address => "127.0.0.1"),
+          Factory.node(:priority => 1, :address => "127.0.0.2"),
+        ],
+      )
+
+      cluster.start_monitoring!
+      @stub_executor.commands.last.should == "ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1"
+    end
+  end
+
+  describe "#monitor_nodes" do
+    it "edit node weight changes in ipvs when the active node is not down" do
+      node1 = Factory.node(:priority => 0, :address => "127.0.0.1", :weight => 90)
+      node3 = Factory.node(:priority => 2, :address => "127.0.0.3", :weight => 88)
+      node2 = Factory.node(:priority => 1, :address => "127.0.0.2", :weight => 87)
+      cluster = Factory.active_passive_cluster(:nodes => [node1, node2, node3], :fwmark => 1)
+      node1.stub(:monitor).and_return(93)
+      node2.stub(:monitor).and_return(92)
+      node3.stub(:monitor).and_return(90)
+
+      cluster.monitor_nodes
+
+      cluster.active_node
+      cluster.active_node.address.should == "127.0.0.1"
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 1 --real-server 127.0.0.1 --ipip --weight 93")
+      node1.weight.should == 93
+    end
+
+    it "replaces active node in ipvs with new weight when the active node is down" do
+      node1 = Factory.node(:priority => 0, :address => "127.0.0.1", :weight => 90)
+      node3 = Factory.node(:priority => 2, :address => "127.0.0.3", :weight => 88)
+      node2 = Factory.node(:priority => 1, :address => "127.0.1.1", :weight => 87)
+      cluster = Factory.active_passive_cluster(:nodes => [node1, node2, node3], :fwmark => 1)
+      node1.stub(:monitor).and_return(0)
+      node2.stub(:monitor).and_return(92)
+      node3.stub(:monitor).and_return(90)
+
+      cluster.monitor_nodes
+
+      cluster.monitor_nodes
+
+      cluster.active_node
+      cluster.active_node.address.should == "127.0.1.1"
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.0.1")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 92")
+    end
+
+    it "replaces the unhealthy least priority node with the next priority node" do
+      node1 = Factory.node(:priority => 0, :address => "127.0.0.1", :weight => 90)
+      node3 = Factory.node(:priority => 2, :address => "127.0.0.3", :weight => 88)
+      node2 = Factory.node(:priority => 1, :address => "127.0.0.2", :weight => 87)
+      cluster = Factory.active_passive_cluster(:nodes => [node1, node2, node3], :fwmark => 1)
+      node1.stub(:monitor).and_return(0)
+      node2.stub(:monitor).and_return(92)
+      node3.stub(:monitor).and_return(90)
+
+      cluster.monitor_nodes
+
+      cluster.active_node
+      cluster.active_node.address.should == "127.0.0.2"
+    end
+
+    it "sets the weight of the current_active_node to 0 in ipvs if all nodes are down" do
+      node1 = Factory.node(:priority => 0, :address => "127.0.0.1", :weight => 90)
+      node3 = Factory.node(:priority => 2, :address => "127.0.0.3", :weight => 88)
+      node2 = Factory.node(:priority => 1, :address => "127.0.0.2", :weight => 87)
+      cluster = Factory.active_passive_cluster(:nodes => [node1, node2, node3], :fwmark => 1)
+      node1.stub(:monitor).and_return(0)
+      node2.stub(:monitor).and_return(0)
+      node3.stub(:monitor).and_return(0)
+
+      cluster.monitor_nodes
+
+      cluster.active_node
+      cluster.active_node.address.should == "127.0.0.1"
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 1 --real-server 127.0.0.1 --ipip --weight 0")
+    end
+  end
+
+  describe "#resume_monitoring!" do
+    it "marks the cluster as monitored" do
+      cluster = Factory.cluster
+
+      cluster.monitored?.should be_false
+      cluster.resume_monitoring!
+      cluster.monitored?.should be_true
+    end
+  end
+
+  describe "synchronize!" do
+    it "continues to monitor clusters that were already monitored" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({})
+      cluster = Factory.cluster(:fwmark => 1)
+
+      cluster.synchronize!
+
+      cluster.should_not be_monitored
+    end
+
+    it "removes current active node if its priority is no longer the least priority" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.1.1']})
+      cluster = Factory.active_passive_cluster(
+        :fwmark => 1,
+        :nodes => [
+          Factory.node(:address => '127.0.1.1', :priority => 8, :weight => 55),
+          Factory.node(:address => '127.0.0.1', :priority => 3, :weight => 75),
+        ],
+      )
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.0.1 --ipip --weight 75")
+    end
+
+    it "removes current active node if the node no longer exist" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.1.1']})
+      cluster = Factory.active_passive_cluster(
+        :fwmark => 1,
+        :nodes => [
+          Factory.node(:address => '127.0.1.2', :priority => 2, :weight => 45),
+          Factory.node(:address => '127.0.0.1', :priority => 3, :weight => 55),
+        ],
+      )
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.2 --ipip --weight 45")
+    end
+
+    it "does not remove current active node if it has the least priority" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.1.1']})
+      cluster = Factory.active_passive_cluster(
+        :fwmark => 1,
+        :nodes => [
+          Factory.node(:address => '127.0.1.1', :priority => 0),
+          Factory.node(:address => '127.0.0.1', :priority => 1),
+        ],
+      )
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should be_empty
+      cluster.active_node.address.should == '127.0.1.1'
+    end
+  end
+end

data/spec/big_brother/app_spec.rb

@@ -8,6 +8,7 @@ module BigBrother
 
     describe "/" do
       it "returns the list of configured clusters and their status" do
+        BigBrother::HealthFetcher.stub(:current_health).and_return(99)
         BigBrother.clusters['one'] = Factory.cluster(:name => 'one', :fwmark => 1)
         BigBrother.clusters['two'] = Factory.cluster(:name => 'two', :fwmark => 2)
         BigBrother.clusters['three'] = Factory.cluster(
@@ -41,9 +42,9 @@ module BigBrother
         BigBrother.clusters['test'] = Factory.cluster(
           :name => 'test',
           :nodes => [
-            Factory.node(:weight => 10),
-            Factory.node(:weight => 20),
-            Factory.node(:weight => 30)
+            Factory.node,
+            Factory.node,
+            Factory.node,
           ]
         )
 
@@ -53,7 +54,7 @@ module BigBrother
         last_response.status.should == 200
         last_response.body.should == <<-RESPONSE_BODY
 Running: true
-CombinedWeight: 60
+CombinedWeight: 300
         RESPONSE_BODY
       end
 
@@ -95,9 +96,9 @@ CombinedWeight: 60
         BigBrother.clusters['test'] = Factory.cluster(
           :name => 'test',
           :nodes => [
-            Factory.node(:weight => 10),
-            Factory.node(:weight => 20),
-            Factory.node(:weight => 30)
+            Factory.node,
+            Factory.node,
+            Factory.node,
           ]
         )
 
@@ -107,7 +108,7 @@ CombinedWeight: 60
         last_response.status.should == 200
         last_response.body.should == <<-RESPONSE_BODY
 Running: true
-CombinedWeight: 60
+CombinedWeight: 300
         RESPONSE_BODY
       end
 
@@ -169,8 +170,8 @@ CombinedWeight: 60
         last_response.body.should == "OK"
         BigBrother.clusters['test'].should be_monitored
         @stub_executor.commands.should include("ipvsadm --add-service --fwmark-service 100 --scheduler wrr")
-        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
-        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 100")
+        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1")
+        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 1")
       end
 
       it "attempts to synchronize the nodes in the cluster" do
@@ -191,13 +192,14 @@ CombinedWeight: 60
         last_response.status.should == 304
         last_response.body.should == ""
         BigBrother.clusters['test'].should be_monitored
-        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.1.225 --ipip --weight 100")
+        @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.1.225 --ipip --weight 0")
         @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 100 --real-server 127.0.1.224")
       end
     end
 
     describe "DELETE /cluster/:name" do
       it "marks the cluster as no longer monitored" do
+        BigBrother::HealthFetcher.stub(:current_health)
         BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
         BigBrother.clusters['test'].start_monitoring!
 

data/spec/big_brother/cluster_collection_spec.rb

@@ -28,6 +28,30 @@ describe BigBrother::ClusterCollection do
       collection.config({'existing_cluster' => cluster_from_config})
     end
 
+    it "ensures an active_active_cluster does a proper cleanup if it is transitioning to a different type of cluster" do
+      collection = BigBrother::ClusterCollection.new
+
+      existing_cluster = Factory.active_active_cluster(:name => 'existing_cluster')
+      collection['existing_cluster'] = existing_cluster
+      cluster_from_config = Factory.cluster(:name => 'existing_cluster')
+
+      existing_cluster.should_receive(:stop_relay_fwmark)
+
+      collection.config({'existing_cluster' => cluster_from_config})
+    end
+
+    it "does not make an active_active_cluster do a clean up if it is being updated" do
+      collection = BigBrother::ClusterCollection.new
+
+      existing_cluster = Factory.active_active_cluster(:name => 'existing_cluster')
+      collection['existing_cluster'] = existing_cluster
+      cluster_from_config = Factory.active_active_cluster(:name => 'existing_cluster')
+
+      existing_cluster.should_not_receive(:stop_relay_fwmark)
+
+      collection.config({'existing_cluster' => cluster_from_config})
+    end
+
     it "stops and removes clusters not included in the next configuration" do
       test2 = Factory.cluster(:name => 'test2', :fwmark => 102)
       collection = BigBrother::ClusterCollection.new
@@ -48,6 +72,7 @@ describe BigBrother::ClusterCollection do
 
   describe "running" do
     it "returns the clusters in the collection that are currently running" do
+      BigBrother::HealthFetcher.stub(:current_health)
       clusters_from_config = {
         'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
         'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),
@@ -65,6 +90,7 @@ describe BigBrother::ClusterCollection do
 
   describe "stopped" do
     it "returns the clusters in the collection that are not running" do
+      BigBrother::HealthFetcher.stub(:current_health)
       clusters_from_config = {
         'test1' => Factory.cluster(:name => 'test1', :fwmark => 101),
         'test2' => Factory.cluster(:name => 'test2', :fwmark => 102),

data/spec/big_brother/cluster_factory_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe BigBrother::ClusterFactory do
+  describe '.create_cluster' do
+    it 'creates a normal cluster' do
+      cluster = BigBrother::ClusterFactory.create_cluster('foo', :fwmark => 100)
+
+      cluster.should be_an_instance_of BigBrother::Cluster
+    end
+
+    it 'creates an active_passive cluster' do
+      cluster = BigBrother::ClusterFactory.create_cluster('foo', :fwmark => 100, :backend_mode => 'active_passive')
+
+      cluster.should be_an_instance_of BigBrother::ActivePassiveCluster
+    end
+
+    it 'creates an active_active cluster' do
+      cluster = BigBrother::ClusterFactory.create_cluster('foo', :fwmark => 100, :backend_mode => 'active_active')
+
+      cluster.should be_an_instance_of BigBrother::ActiveActiveCluster
+    end
+  end
+end

data/spec/big_brother/cluster_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe BigBrother::Cluster do
+  before { BigBrother::HealthFetcher.stub(:current_health).and_return(10) }
+
   describe "#start_monitoring!" do
     it "marks the cluster as monitored" do
       cluster = Factory.cluster
@@ -32,17 +34,16 @@ describe BigBrother::Cluster do
     it "invalidates recorded weights, so it properly updates after a stop/start" do
       node = Factory.node(:address => '127.0.0.1')
       cluster = Factory.cluster(:fwmark => '100', :nodes => [node])
-
-      BigBrother::HealthFetcher.stub(:current_health).and_return(10)
-
       cluster.start_monitoring!
       cluster.monitor_nodes
 
       cluster.stop_monitoring!
       cluster.start_monitoring!
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(15)
       cluster.monitor_nodes
 
-      @stub_executor.commands.last.should == "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 10"
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 15")
     end
   end
 
@@ -56,6 +57,46 @@ describe BigBrother::Cluster do
   end
 
   describe "#monitor_nodes" do
+    it "does not run multiple ipvsadm commands if the health does not change" do
+      node = Factory.node(:address => '127.0.0.1')
+      cluster = Factory.cluster(:fwmark => 100, :nodes => [node])
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should == ["ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56"]
+    end
+
+    it "will run multiple ipvsadm commands if the health does change" do
+      node = Factory.node(:address => '127.0.0.1')
+      cluster = Factory.cluster(:fwmark => 100, :nodes => [node])
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:current_health).and_return(41)
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should == [
+        "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56",
+        "ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 41"
+      ]
+    end
+
+    it "does not update the weight if the cluster is no longer monitored" do
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      node = Factory.node(:address => '127.0.0.1')
+      cluster = Factory.cluster(:fwmark => 100, :nodes => [node])
+      cluster.stop_monitoring!
+
+      @stub_executor.commands.clear
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should == []
+    end
+
     it "marks the cluster as no longer requiring monitoring" do
       cluster = Factory.cluster
 
@@ -71,6 +112,7 @@ describe BigBrother::Cluster do
       node1 = Factory.node
       node2 = Factory.node
       cluster = Factory.cluster(:nodes => [node1, node2])
+      cluster.start_monitoring!
 
       node1.should_receive(:monitor).with(cluster)
       node2.should_receive(:monitor).with(cluster)
@@ -112,7 +154,7 @@ describe BigBrother::Cluster do
       cluster.start_monitoring!
       cluster.monitor_nodes
 
-      @stub_executor.commands.last.should == "ipvsadm --add-server --fwmark-service 1 --real-server 127.0.0.1 --ipip --weight 1"
+      @stub_executor.commands.last.should == "ipvsadm --add-server --fwmark-service 1 --real-server 169.254.254.254 --ipip --weight 1"
     end
 
     it "removes downpage node from IPVS if it exists and cluster is up" do
@@ -137,8 +179,8 @@ describe BigBrother::Cluster do
         node2 = Factory.node(:address => '192.168.0.2')
         cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
 
-        node1.stub(:_determine_weight).and_return(0)
-        node2.stub(:_determine_weight).and_return(10)
+        node1.stub(:monitor).and_return(0)
+        node2.stub(:monitor).and_return(10)
 
         cluster.start_monitoring!
         cluster.monitor_nodes
@@ -150,8 +192,8 @@ describe BigBrother::Cluster do
         node2 = Factory.node(:address => '192.168.0.2')
         cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
 
-        node1.stub(:_determine_weight).and_return(0)
-        node2.stub(:_determine_weight).and_return(10)
+        node1.stub(:monitor).and_return(0)
+        node2.stub(:monitor).and_return(10)
 
         cluster.start_monitoring!
         cluster.monitor_nodes
@@ -167,9 +209,9 @@ describe BigBrother::Cluster do
         node3 = Factory.node(:address => '192.168.0.3')
         cluster = Factory.cluster(:nodes => [node1, node2, node3], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
 
-        node1.stub(:_determine_weight).and_return(0)
-        node2.stub(:_determine_weight).and_return(10)
-        node3.stub(:_determine_weight).and_return(10)
+        node1.stub(:monitor).and_return(0)
+        node2.stub(:monitor).and_return(10)
+        node3.stub(:monitor).and_return(10)
 
         cluster.start_monitoring!
         cluster.monitor_nodes
@@ -180,16 +222,16 @@ describe BigBrother::Cluster do
         node1 = Factory.node(:address => '192.168.0.1')
         node2 = Factory.node(:address => '192.168.0.2')
         cluster = Factory.cluster(:nodes => [node1, node2], :nagios => {:host => "prod.load", :check => "test1_check", :server => "server.foo"})
-        node1.stub(:_determine_weight).and_return(0)
-        node2.stub(:_determine_weight).and_return(10)
+        node1.stub(:monitor).and_return(0)
+        node2.stub(:monitor).and_return(10)
 
         cluster.start_monitoring!
         cluster.monitor_nodes
         @stub_executor.commands.should include("echo 'prod.load,test1_check,2,CRITICAL 50% of nodes are down' | send_nsca -H server.foo -d ,")
         @stub_executor.clear_commands!
 
-        node1.stub(:_determine_weight).and_return(10)
-        node2.stub(:_determine_weight).and_return(10)
+        node1.stub(:monitor).and_return(10)
+        node2.stub(:monitor).and_return(10)
         cluster.monitor_nodes
         @stub_executor.commands.should include("echo 'prod.load,test1_check,0,OK all nodes up' | send_nsca -H server.foo -d ,")
       end
@@ -208,7 +250,7 @@ describe BigBrother::Cluster do
   end
 
   describe "synchronize!" do
-    it "monitors clusters that were already monitored" do
+    it "continues to monitor clusters that were already monitored" do
       BigBrother.ipvs.stub(:running_configuration).and_return('1' => ['127.0.0.1'])
       cluster = Factory.cluster(:fwmark => 1)
 
@@ -250,7 +292,7 @@ describe BigBrother::Cluster do
 
       cluster.synchronize!
 
-      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 100")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 0")
     end
   end