big_brother 0.6.8 → 0.8.7

data/lib/big_brother/health_fetcher.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module BigBrother
   class HealthFetcher
     def self.current_health(address, port, path)
@@ -8,6 +10,16 @@ module BigBrother
       response.response_header.status == 200 ? _parse_health(response) : 0
     end
 
+    def self.interpol_status(interpol_node, fwmark)
+      url = "http://#{interpol_node.address}:#{interpol_node.port}/fwmark/#{fwmark}"
+
+      BigBrother.logger.debug("Fetching health from #{url}")
+      response = EventMachine::HttpRequest.new(url).get
+      response.response_header.status == 200 ? JSON.parse(response.response) : []
+    rescue JSON::ParserError
+      []
+    end
+
     def self._parse_health(response)
       if response.response_header.has_key?('X_HEALTH')
         response.response_header['X_HEALTH'].to_i

data/lib/big_brother/node.rb

@@ -2,7 +2,9 @@ require 'net/http'
 
 module BigBrother
   class Node
-    attr_reader :address, :port, :path, :start_time, :weight
+    attr_reader :address, :port, :path, :start_time, :priority, :max_weight
+    attr_accessor :weight, :down_tick_count
+    INITIAL_WEIGHT = 1
 
     def initialize(attributes={})
       @address = attributes[:address]
@@ -10,6 +12,10 @@ module BigBrother
       @path = attributes[:path]
       @weight = attributes[:weight]
       @start_time = attributes.fetch(:start_time, Time.now.to_i)
+      @priority = attributes.fetch(:priority, 0)
+      @interpol = attributes.fetch(:interpol, false)
+      @max_weight = attributes[:max_weight]
+      @down_tick_count = 0
     end
 
     def age
@@ -27,26 +33,45 @@ module BigBrother
       @weight = nil
     end
 
-    def monitor(cluster)
-      new_weight = _determine_weight(cluster)
-      return unless cluster.monitored?
-      if new_weight != @weight
-        BigBrother.ipvs.edit_node(cluster.fwmark, address, new_weight)
-        @weight = new_weight
-      end
+    def interpol?
+      @interpol
     end
 
     def ==(other)
       address == other.address && port == other.port
     end
+    alias eql? ==
+
+    def hash
+      [@address, @port].hash
+    end
+
+    def <=>(other)
+      return 1 if self.weight.to_i.zero?
+      return -1 if other.weight.to_i.zero?
+      comparison = self.priority <=> other.priority
+      if comparison.zero?
+       self.address <=> other.address
+      else
+        comparison
+      end
+    end
 
-    def _determine_weight(cluster)
+    def monitor(cluster)
       if cluster.up_file_exists?
         100
       elsif cluster.down_file_exists?
         0
       else
-        _weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time)
+        _cap_weight(_weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time))
+      end
+    end
+
+    def _cap_weight(health)
+      if !@max_weight.nil? && @max_weight.is_a?(Integer) && @max_weight > 0 && @max_weight < health
+        @max_weight
+      else
+        health
       end
     end
 

data/lib/big_brother/version.rb

@@ -1,3 +1,3 @@
 module BigBrother
-  VERSION = "0.6.8"
+  VERSION = "0.8.7"
 end

data/lib/resources/config_schema.yml

@@ -0,0 +1,92 @@
+type:                           map
+mapping:
+  "_big_brother":
+    type:                       map
+    mapping:
+      "offset":
+        type:                   int
+      "check_interval":
+        type:                   int
+      "max_down_ticks":
+        type:                   int
+      "backend_mode":
+        type:                   str
+        enum:                   [active_passive, active_active]
+      "scheduler":
+        type:                   str
+        enum:                   [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
+      "ramp_up_time":
+        type:                   int
+      "has_downpage":
+        type:                   bool
+      "nagios":
+        type:                   map
+        mapping:
+          "server":
+            type:               str
+          "check":
+            type:               str
+          "host":
+            type:               str
+  "clusters":
+     type:                      seq
+     required:                  yes
+     sequence:
+       - type:                  map
+         mapping:
+           "non_egress_locations":
+             type:              seq
+             sequence:
+               - type:          str
+           "cluster_name":
+             type:              str
+             required:          yes
+           "check_interval":
+             type:              int
+           "max_down_ticks":
+             type:              int
+           "backend_mode":
+             type:              str
+             enum:              [active_passive, active_active]
+           "scheduler":
+             type:              str
+             enum:              [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
+             required:          yes
+           "fwmark":
+             type:              int
+             required:          yes
+           "ramp_up_time":
+             type:              int
+           "has_downpage":
+             type:              bool
+           "nagios":
+             type:              map
+             mapping:
+               "server":
+                 type:          str
+               "check":
+                 type:          str
+               "host":
+                 type:          str
+           "nodes":
+             type:              seq
+             required:          yes
+             sequence:
+               - type:          map
+                 mapping:
+                   "address":
+                     type:      str
+                     pattern:   /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
+                     required:  yes
+                   "interpol":
+                     type:      bool
+                   "max_weight":
+                     type:      int
+                   "priority":
+                     type:      int
+                   "port":
+                     type:      int
+                     range:     { max: 65535, min: 1 }
+                   "path":
+                     type:      str
+                     pattern:   /^\//

data/spec/big_brother/active_active_cluster_spec.rb

@@ -0,0 +1,437 @@
+require 'spec_helper'
+
+describe BigBrother::ActiveActiveCluster do
+  before { BigBrother::HealthFetcher.stub(:current_health).and_return(10) }
+
+  describe '#start_monitoring!' do
+    it 'starts all non interpol nodes' do
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          Factory.node(:interpol => true,  :address => '127.0.0.3'),
+        ],
+      )
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      cluster.start_monitoring!
+
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1')
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 1')
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.3 --ipip --weight 1')
+    end
+
+    it 'starts all relay fwmark service' do
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :offset => 10000,
+        :nodes => [
+          Factory.node(:interpol => true,  :address => '127.0.0.3'),
+        ],
+      )
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      cluster.start_monitoring!
+
+      @stub_executor.commands.should include('ipvsadm --add-service --fwmark-service 10100 --scheduler wrr')
+    end
+
+    it 'starts all local_nodes on relay fwmark' do
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :offset => 10000,
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          Factory.node(:interpol => true,  :address => '127.0.0.3'),
+        ],
+      )
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      cluster.start_monitoring!
+
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 1')
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.2 --ipip --weight 1')
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.3 --ipip --weight 1')
+    end
+
+    it 'does not attempt to retrieve interpol status if there is no interpol node' do
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+        ],
+      )
+
+      BigBrother::HealthFetcher.should_not_receive(:interpol_status)
+
+      cluster.start_monitoring!
+    end
+
+    it 'does not start interpol nodes when the remote relay cluster is non-existent' do
+      node = Factory.node(:interpol => true,  :address => '127.0.0.3')
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          node,
+        ],
+      )
+
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([])
+      cluster.start_monitoring!
+
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
+    end
+
+    it 'does not start interpol nodes when the remote regular cluster is non-existent' do
+      node = Factory.node(:interpol => true,  :address => '127.0.0.3')
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          node,
+        ],
+      )
+
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([])
+      cluster.start_monitoring!
+
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
+    end
+
+    it 'starts all interpol nodes if both the regular and relay remote clusters exist' do
+      node = Factory.node(:interpol => true,  :address => '127.0.0.3')
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          node,
+        ],
+      )
+
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
+    end
+
+    it 'does not start interpol nodes if their "lb_source_location" is included in the "non_egress_locations"' do
+      node = Factory.node(:interpol => true,  :address => '127.0.0.3')
+      cluster = Factory.active_active_cluster(
+        :fwmark => 100,
+        :scheduler => 'wrr',
+        :non_egress_locations => ['test'],
+        :nodes => [
+          Factory.node(:interpol => false, :address => '127.0.0.1'),
+          Factory.node(:interpol => false, :address => '127.0.0.2'),
+          node,
+        ],
+      )
+
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
+      BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
+      @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 55')
+    end
+  end
+
+  describe "#monitor_nodes" do
+    it "update weight of local ipvs nodes" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:current_health).and_return(41)
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 41")
+    end
+
+    it "update weight of relay ipvs nodes" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :offset => 10000, :nodes => [node, interpol_node])
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:current_health).and_return(41)
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56")
+       @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 41")
+    end
+
+    it "does not update remote nodes for relay fwmark" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 91,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :offset => 10_000)
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.resume_monitoring!
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should_not include("ipvsadm --edit-server --fwmark-service 10100 --real-server 172.27.3.1 --ipip --weight 56")
+    end
+
+    it "update weight of remote nodes" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+
+      cluster.remote_nodes.first.weight.should == 45
+
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
+    end
+
+    it "does not update weight of remote nodes if the weight has not changed" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 55}])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should == ["ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45"]
+    end
+
+    it "update weight of remote not returned to 0" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 2)
+      cluster.start_monitoring!
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 0")
+    end
+
+    it "adds newly discovered remote nodes to ipvs" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
+      cluster.start_monitoring!
+      cluster.instance_variable_get(:@remote_nodes).size.should == 0
+
+      @stub_executor.commands.clear
+
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      BigBrother::HealthFetcher.stub(:current_health).and_return(56)
+      cluster.monitor_nodes
+
+      cluster.remote_nodes.size.should == 1
+
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return(
+        [
+          {'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65},
+          {'aggregated_health' => 100,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 40},
+        ]
+      )
+      cluster.monitor_nodes
+
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45")
+      @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 40")
+      cluster.instance_variable_get(:@remote_nodes).size.should == 2
+    end
+
+    it "removes a remote node from ipvs if it has been down for too many ticks" do
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+      node = Factory.node(:address => '127.0.0.1')
+      interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
+      cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 100)
+      cluster.start_monitoring!
+      cluster.monitor_nodes
+      @stub_executor.commands.clear
+
+      (cluster.max_down_ticks + 1).times do
+        BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
+        cluster.monitor_nodes
+      end
+
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 100 --real-server 172.27.3.1")
+      cluster.instance_variable_get(:@remote_nodes).should be_empty
+    end
+  end
+
+  describe "#synchronize!" do
+    it "does not remove remote nodes" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
+      cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should be_empty
+    end
+
+    it "does not append remote_nodes to nodes" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
+      cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
+
+      cluster.synchronize!
+
+      cluster.nodes.size.should == 1
+    end
+  end
+
+  describe "synchronize!" do
+    it "continues to monitor clusters that were already monitored" do
+      BigBrother.ipvs.stub(:running_configuration).and_return('1' => ['127.0.0.1'])
+      cluster = Factory.active_active_cluster(:fwmark => 1)
+
+      cluster.synchronize!
+
+      cluster.should be_monitored
+    end
+
+    it "does not monitor clusters that were already monitored" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({})
+      cluster = Factory.active_active_cluster(:fwmark => 1)
+
+      cluster.synchronize!
+
+      cluster.should_not be_monitored
+    end
+
+    it "does not attempt to re-add the services it was monitoring" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should be_empty
+    end
+
+    it "removes relay nodes that are no longer part of the cluster" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '127.0.1.1']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1')])
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1")
+      @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1")
+    end
+
+    it "adds new relay nodes to the cluster" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1")])
+
+      cluster.synchronize!
+
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 0")
+      @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 0")
+    end
+
+    it "does not add remote nodes to relay ipvs fwmark" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
+      cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
+      BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
+
+      cluster.start_monitoring!
+      cluster.synchronize!
+
+      @stub_executor.commands.should_not include("ipvsadm --add-server --fwmark-service 10001 --real-server 172.27.3.1 --ipip --weight 0")
+    end
+  end
+
+  describe "#stop_monitoring!" do
+    it "deletes the relay fwmark" do
+      cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [])
+      cluster.stop_monitoring!
+
+      @stub_executor.commands.should include("ipvsadm --delete-service --fwmark-service 10001")
+    end
+  end
+
+  describe "#incorporate_state" do
+    it "starts a relay_fwmark when it is not started" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
+      active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
+      cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
+
+      active_active_cluster.incorporate_state(cluster)
+
+      @stub_executor.commands.should == ["ipvsadm --add-service --fwmark-service 10001 --scheduler wrr", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.0.1 --ipip --weight 80", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 100"]
+    end
+
+    it "does not starts a relay_fwmark when the cluster is not running" do
+      BigBrother.ipvs.stub(:running_configuration).and_return({'2' => ['127.0.0.1']})
+      active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
+      cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
+
+      active_active_cluster.incorporate_state(cluster)
+
+      @stub_executor.commands.should be_empty
+    end
+  end
+
+  describe "#stop_relay_fwmark" do
+    it "stops relay fwmark and all nodes in the relay fwmark" do
+      active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
+
+      active_active_cluster.stop_relay_fwmark
+
+      @stub_executor.commands.should == ["ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.0.1", "ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1", "ipvsadm --delete-service --fwmark-service 10001"]
+    end
+  end
+end