big_brother 0.6.8 → 0.8.7

data/.travis.yml

@@ -2,3 +2,4 @@ language: ruby
 rvm:
   - 1.9.2
   - 1.9.3
+sudo: false

data/Changelog.md

@@ -0,0 +1,5 @@
+0.6.8:
+
+- Enable active/passive mode, an optional `backend_mode: "active_passive"` configuration directive is added to the big brother cluster config.
+  - Active/Passive nodes now take a `priority` field that is used to select the active backend for the cluster. 
+  - The active backend for an active/passive cluster is the backend with the lowest priority and positive health response. Should two backends have the same priority, the lowest backend IP address is selected. If none of the backends is healthy, the last active node's health is updated to weight of 0 in ipvs.

data/big_brother.gemspec

@@ -24,10 +24,10 @@ Gem::Specification.new do |gem|
   gem.add_dependency "em-synchrony",      "~> 1.0"
   gem.add_dependency "em-resolv-replace", "~> 1.1"
   gem.add_dependency "em-syslog",         "~> 0.0.2"
+  gem.add_dependency "kwalify",           "~> 0.7.2"
 
   gem.add_development_dependency "rspec",       "~> 2.9.0"
   gem.add_development_dependency "rack-test",   "~> 0.6.1"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rake_commit", "~> 0.13"
-  gem.add_development_dependency "vagrant"
 end

data/lib/big_brother.rb

@@ -4,11 +4,15 @@ require 'em-synchrony/em-http'
 require 'em/syslog'
 require 'thin'
 require 'yaml'
+require 'kwalify'
 
 require 'sinatra/synchrony'
 
 require 'big_brother/app'
 require 'big_brother/cluster'
+require 'big_brother/active_passive_cluster'
+require 'big_brother/active_active_cluster'
+require 'big_brother/cluster_factory'
 require 'big_brother/cluster_collection'
 require 'big_brother/configuration'
 require 'big_brother/health_fetcher'

data/lib/big_brother/active_active_cluster.rb

@@ -0,0 +1,157 @@
+module BigBrother
+  class ActiveActiveCluster < BigBrother::Cluster
+    attr_reader :interpol_node, :max_down_ticks, :offset, :remote_nodes, :local_nodes, :non_egress_locations
+
+    def initialize(name, attributes={})
+      super(name, attributes)
+      interpol_nodes, local_nodes = @nodes.partition { |node| node.interpol? }
+      @nodes = @local_nodes = local_nodes
+      @interpol_node = interpol_nodes.first
+      @remote_nodes = []
+      @max_down_ticks = attributes.fetch(:max_down_ticks, 0)
+      @offset = attributes.fetch(:offset, 10_000)
+      @non_egress_locations = *attributes.fetch(:non_egress_locations, [])
+    end
+
+    def start_monitoring!
+      BigBrother.logger.info "Starting monitoring on active/active cluster #{to_s}"
+      BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
+      BigBrother.ipvs.start_cluster(_relay_fwmark, @scheduler)
+      local_nodes.each do |node|
+        BigBrother.ipvs.start_node(@fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
+        BigBrother.ipvs.start_node(_relay_fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
+      end
+
+      @monitored = true
+    end
+
+    def stop_monitoring!
+      super
+      BigBrother.ipvs.stop_cluster(_relay_fwmark)
+    end
+
+    def monitor_nodes
+      super
+
+      fresh_remote_nodes = _fetch_remote_nodes
+      remote_nodes.each do |node|
+        if new_node = fresh_remote_nodes[node.address]
+          next if new_node.weight == node.weight
+          BigBrother.ipvs.edit_node(fwmark, node.address, new_node.weight)
+          node.weight = new_node.weight
+        else
+          _adjust_or_remove_remote_node(node)
+        end
+      end
+
+      _add_remote_nodes(fresh_remote_nodes.values - remote_nodes)
+    end
+
+    def synchronize!
+      ipvs_state = BigBrother.ipvs.running_configuration
+      @remote_nodes = _fetch_remote_nodes.values if @remote_nodes == []
+      if ipvs_state.has_key?(fwmark.to_s)
+        resume_monitoring!
+
+        running_nodes = ipvs_state[fwmark.to_s]
+        _remove_nodes(running_nodes - cluster_nodes)
+        _add_nodes(cluster_nodes - running_nodes, fwmark)
+        _add_nodes(local_cluster_nodes - running_nodes, _relay_fwmark)
+      end
+    end
+
+    def cluster_nodes
+      (nodes + remote_nodes).map(&:address)
+    end
+
+    def local_cluster_nodes
+      nodes.map(&:address)
+    end
+
+    def incorporate_state(cluster)
+      ipvs_state = BigBrother.ipvs.running_configuration
+      if ipvs_state[fwmark.to_s] && ipvs_state[_relay_fwmark.to_s].nil?
+        BigBrother.logger.info "Adding new active/active LB node #{to_s}"
+        BigBrother.ipvs.start_cluster(_relay_fwmark, @scheduler)
+      end
+
+      if ipvs_state[fwmark.to_s] && ipvs_state.fetch(_relay_fwmark.to_s, []).empty?
+        nodes.each do |node|
+          actual_node = cluster.find_node(node.address, node.port)
+          BigBrother.ipvs.start_node(_relay_fwmark, actual_node.address, actual_node.weight)
+        end
+      end
+
+      BigBrother.logger.info "Merging in new active/active cluster #{to_s}"
+
+      @remote_nodes = cluster.remote_nodes if cluster.is_a?(BigBrother::ActiveActiveCluster)
+
+      super(cluster)
+    end
+
+    def stop_relay_fwmark
+      nodes.each do |node|
+        BigBrother.ipvs.stop_node(_relay_fwmark, node.address)
+      end
+
+      BigBrother.ipvs.stop_cluster(_relay_fwmark)
+    end
+
+    def _relay_fwmark
+      fwmark + offset
+    end
+
+    def _add_nodes(addresses, fwmark)
+      addresses.each do |address|
+        BigBrother.logger.info "Adding #{address} to active/active cluster #{self}"
+        BigBrother.ipvs.start_node(fwmark, address, 0)
+      end
+    end
+
+    def _adjust_or_remove_remote_node(node)
+      if node.down_tick_count >= max_down_ticks
+        BigBrother.ipvs.stop_node(fwmark, node.address)
+        remote_nodes.delete(node)
+      else
+        BigBrother.ipvs.edit_node(fwmark, node.address, 0)
+        node.weight = 0
+        node.down_tick_count += 1
+      end
+    end
+
+    def _fetch_remote_nodes
+      return {} if interpol_node.nil?
+
+      regular_remote_cluster = BigBrother::HealthFetcher.interpol_status(interpol_node, fwmark)
+      relay_remote_cluster = BigBrother::HealthFetcher.interpol_status(interpol_node, _relay_fwmark)
+
+      return {} if regular_remote_cluster.empty? || relay_remote_cluster.empty?
+
+      regular_remote_cluster.each_with_object({}) do |node, hsh|
+        next if self.non_egress_locations.include?(node['lb_source_location'])
+
+        hsh[node['lb_ip_address']] = BigBrother::Node.new(:address => node['lb_ip_address'], :weight => node['health'])
+      end
+    end
+
+    def _add_remote_nodes(nodes)
+      nodes.each do |node|
+        BigBrother.ipvs.start_node(fwmark, node.address, node.weight)
+        @remote_nodes << node
+      end
+    end
+
+    def _remove_nodes(addresses)
+      addresses.each do |address|
+        BigBrother.logger.info "Removing #{address} from active/active cluster #{self}"
+        BigBrother.ipvs.stop_node(fwmark, address)
+        BigBrother.ipvs.stop_node(_relay_fwmark, address)
+      end
+    end
+
+    def _update_node(node, new_weight)
+      BigBrother.ipvs.edit_node(fwmark, node.address, new_weight)
+      BigBrother.ipvs.edit_node(_relay_fwmark, node.address, new_weight)
+    end
+  end
+end

data/lib/big_brother/active_passive_cluster.rb

@@ -0,0 +1,61 @@
+module BigBrother
+  class ActivePassiveCluster < Cluster
+
+    def start_monitoring!
+      BigBrother.logger.info "starting monitoring on cluster #{to_s}"
+      BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
+      BigBrother.ipvs.start_node(@fwmark, active_node.address, BigBrother::Node::INITIAL_WEIGHT)
+
+      @monitored = true
+    end
+
+    def active_node
+      @current_active_node ||= @nodes.sort.first
+    end
+
+    def synchronize!
+      ipvs_state = BigBrother.ipvs.running_configuration
+      if ipvs_state.has_key?(fwmark.to_s)
+        resume_monitoring!
+
+        running_active_node_address = ipvs_state[fwmark.to_s].first
+        if running_active_node_address != active_node.address
+          BigBrother.ipvs.stop_node(fwmark, running_active_node_address)
+          BigBrother.ipvs.start_node(fwmark, active_node.address, active_node.weight)
+        end
+      end
+    end
+
+    def monitor_nodes
+      @last_check = Time.now
+      @current_active_node = active_node
+      proposed_active_node = @nodes.reject do |node|
+        weight = node.monitor(self).to_i
+        _modify_current_active_node_weight(node, weight)
+        node.weight = weight
+        node.weight.zero?
+      end.sort.first
+
+      if proposed_active_node != @current_active_node && !proposed_active_node.nil?
+        _modify_active_node(@current_active_node, proposed_active_node)
+        @current_active_node = proposed_active_node
+      end
+
+      _check_downpage if has_downpage?
+      _notify_nagios if nagios
+    end
+
+    def _modify_current_active_node_weight(node, weight)
+      return unless node == @current_active_node
+      if @current_active_node.weight != weight
+        BigBrother.ipvs.edit_node(fwmark, @current_active_node.address, weight)
+        @current_active_node.weight = weight
+      end
+    end
+
+    def _modify_active_node(current_active_node, proposed_active_node)
+      BigBrother.ipvs.stop_node(fwmark, current_active_node.address)
+      BigBrother.ipvs.start_node(fwmark, proposed_active_node.address, proposed_active_node.weight)
+    end
+  end
+end

data/lib/big_brother/cluster.rb

@@ -1,6 +1,6 @@
 module BigBrother
   class Cluster
-    attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name, :ramp_up_time, :nagios
+    attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name, :ramp_up_time, :nagios, :backend_mode
 
     def initialize(name, attributes = {})
       @name = name
@@ -15,6 +15,7 @@ module BigBrother
       @ramp_up_time = attributes.fetch(:ramp_up_time, 60)
       @has_downpage = attributes[:has_downpage]
       @nagios = attributes[:nagios]
+      @backend_mode = attributes[:backend_mode]
     end
 
     def _coerce_node(node_config)
@@ -42,17 +43,17 @@ module BigBrother
     end
 
     def start_monitoring!
-      BigBrother.logger.info "starting monitoring on cluster #{to_s}"
+      BigBrother.logger.info "Starting monitoring on cluster #{to_s}"
       BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
       @nodes.each do |node|
-        BigBrother.ipvs.start_node(@fwmark, node.address, 100)
+        BigBrother.ipvs.start_node(@fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
       end
 
       @monitored = true
     end
 
     def stop_monitoring!
-      BigBrother.logger.info "stopping monitoring on cluster #{to_s}"
+      BigBrother.logger.info "Stopping monitoring on cluster #{to_s}"
       BigBrother.ipvs.stop_cluster(@fwmark)
 
       @monitored = false
@@ -60,7 +61,7 @@ module BigBrother
     end
 
     def resume_monitoring!
-      BigBrother.logger.info "resuming monitoring on cluster #{to_s}"
+      BigBrother.logger.info "Resuming monitoring on cluster #{to_s}"
       @monitored = true
     end
 
@@ -70,13 +71,15 @@ module BigBrother
         resume_monitoring!
 
         running_nodes = ipvs_state[fwmark.to_s]
-        cluster_nodes = nodes.map(&:address)
-
         _remove_nodes(running_nodes - cluster_nodes)
         _add_nodes(cluster_nodes - running_nodes)
       end
     end
 
+    def cluster_nodes
+      nodes.map(&:address)
+    end
+
     def needs_check?
       return false unless monitored?
       @last_check + @check_interval < Time.now
@@ -84,7 +87,14 @@ module BigBrother
 
     def monitor_nodes
       @last_check = Time.now
-      @nodes.each { |node| node.monitor(self) }
+      return unless monitored?
+      @nodes.each do |node|
+        new_weight = node.monitor(self)
+        if new_weight != node.weight
+          _update_node(node, new_weight)
+          node.weight = new_weight
+        end
+      end
 
       _check_downpage if has_downpage?
       _notify_nagios if nagios
@@ -110,19 +120,20 @@ module BigBrother
       nodes.each do |node|
         node.incorporate_state(another_cluster.find_node(node.address, node.port))
       end
+
       self
     end
 
     def _add_nodes(addresses)
       addresses.each do |address|
-        BigBrother.logger.info "adding #{address} to cluster #{self}"
-        BigBrother.ipvs.start_node(fwmark, address, 100)
+        BigBrother.logger.info "Adding #{address} to cluster #{self}"
+        BigBrother.ipvs.start_node(fwmark, address, 0)
       end
     end
 
     def _add_maintenance_node
-      BigBrother.logger.info "adding 127.0.0.1 to cluster #{self}"
-      BigBrother.ipvs.start_node(fwmark, '127.0.0.1', 1)
+      BigBrother.logger.info "Adding downpage to cluster #{self}"
+      BigBrother.ipvs.start_node(fwmark, '169.254.254.254', 1)
     end
 
     def _check_downpage
@@ -155,9 +166,13 @@ module BigBrother
 
     def _remove_nodes(addresses)
       addresses.each do |address|
-        BigBrother.logger.info "removing #{address} to cluster #{self}"
+        BigBrother.logger.info "Removing #{address} to cluster #{self}"
         BigBrother.ipvs.stop_node(fwmark, address)
       end
     end
+
+    def _update_node(node, new_weight)
+      BigBrother.ipvs.edit_node(fwmark, node.address, new_weight)
+    end
   end
 end

data/lib/big_brother/cluster_collection.rb

@@ -15,6 +15,9 @@ module BigBrother
       end
       new_clusters.each do |cluster_name, cluster|
         if @clusters.key?(cluster_name)
+          current_cluster = @clusters[cluster_name]
+          current_cluster.stop_relay_fwmark if current_cluster.is_a?(BigBrother::ActiveActiveCluster) && !cluster.is_a?(BigBrother::ActiveActiveCluster)
+
           @clusters[cluster_name] = cluster.incorporate_state(@clusters[cluster_name])
         else
           @clusters[cluster_name] = cluster

data/lib/big_brother/cluster_factory.rb

@@ -0,0 +1,16 @@
+module BigBrother
+  class ClusterFactory
+
+    ACTIVE_PASSIVE_CLUSTER = 'active_passive'
+    ACTIVE_ACTIVE_CLUSTER = 'active_active'
+
+    CLUSTERS = {
+      ACTIVE_PASSIVE_CLUSTER => ActivePassiveCluster,
+      ACTIVE_ACTIVE_CLUSTER  => ActiveActiveCluster,
+    }
+
+    def self.create_cluster(name, attributes)
+      CLUSTERS.fetch(attributes[:backend_mode], BigBrother::Cluster).new(name, attributes)
+    end
+  end
+end

data/lib/big_brother/configuration.rb

@@ -4,11 +4,15 @@ module BigBrother
 
     def self.from_file(config_file)
       config = YAML.load_file(config_file)
+      return {} unless _valid?(config)
+
+      configured_clusters = config.fetch("clusters")
       defaults = config.delete(GLOBAL_CONFIG_KEY)
 
-      config.inject({}) do |clusters, (cluster_name, cluster_values)|
-        cluster_details = _apply_defaults(defaults, cluster_values)
-        clusters.merge(cluster_name => Cluster.new(cluster_name, _deeply_symbolize_keys(cluster_details)))
+      configured_clusters.inject({}) do |clusters, cluster|
+        cluster_details = _apply_defaults(defaults, cluster)
+        cluster_name    = cluster.fetch("cluster_name")
+        clusters.merge(cluster_name => BigBrother::ClusterFactory.create_cluster(cluster_name, _deeply_symbolize_keys(cluster_details)))
       end
     end
 
@@ -31,5 +35,19 @@ module BigBrother
         oldval.is_a?(Hash) && newval.is_a?(Hash) ? _apply_defaults(oldval, newval) : newval
       end
     end
+
+    def self._valid?(config)
+      schema_path = File.join(File.dirname(__FILE__), "../resources", "config_schema.yml")
+      schema = YAML.load_file(schema_path)
+      validator = Kwalify::Validator.new(schema)
+      errors = validator.validate(config)
+      valid = !(errors && !errors.empty?)
+
+      unless valid
+        errors.each { |err| BigBrother.logger.info("- [#{err.path}] #{err.message}") }
+      end
+
+      valid
+    end
   end
 end