big_brother 0.1.0

data/lib/big_brother/cluster.rb

@@ -0,0 +1,70 @@
+module BigBrother
+  class Cluster
+    attr_reader :fwmark, :scheduler, :check_interval, :nodes
+
+    def initialize(name, attributes = {})
+      @name = name
+      @fwmark = attributes['fwmark']
+      @scheduler = attributes['scheduler']
+      @check_interval = attributes.fetch('check_interval', 1)
+      @monitored = false
+      @nodes = attributes.fetch('nodes', [])
+      @last_check = Time.new(0)
+      @up_file = BigBrother::StatusFile.new('up', @name)
+      @down_file = BigBrother::StatusFile.new('down', @name)
+    end
+
+    def monitored?
+      @monitored
+    end
+
+    def start_monitoring!
+      BigBrother.logger.info "starting monitoring on cluster #{to_s}"
+      BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
+      @nodes.each do |node|
+        BigBrother.ipvs.start_node(@fwmark, node.address, 100)
+      end
+
+      @monitored = true
+    end
+
+    def stop_monitoring!
+      BigBrother.logger.info "stopping monitoring on cluster #{to_s}"
+      BigBrother.ipvs.stop_cluster(@fwmark)
+
+      @monitored = false
+    end
+
+    def resume_monitoring!
+      BigBrother.logger.info "resuming monitoring on cluster #{to_s}"
+      @monitored = true
+    end
+
+    def needs_check?
+      return false unless monitored?
+      @last_check + @check_interval < Time.now
+    end
+
+    def monitor_nodes
+      @nodes.each do |node|
+        BigBrother.ipvs.edit_node(@fwmark, node.address, _determine_weight(node))
+      end
+
+      @last_check = Time.now
+    end
+
+    def _determine_weight(node)
+      if @up_file.exists?
+        100
+      elsif @down_file.exists?
+        0
+      else
+        node.current_health
+      end
+    end
+
+    def to_s
+      "#{@name} (#{@fwmark})"
+    end
+  end
+end

data/lib/big_brother/configuration.rb

@@ -0,0 +1,48 @@
+module BigBrother
+  class Configuration
+    def self.evaluate(file)
+      yaml = YAML.load(File.read(file))
+      assoc_array = yaml.map do |name, values|
+        nodes = _parse_nodes(values.delete('nodes'))
+        [name, Cluster.new(name, values.merge('nodes' => nodes))]
+      end
+
+      Hash[assoc_array]
+    end
+
+    def self.synchronize_with_ipvs(clusters, ipvs_state)
+      clusters.values.each do |cluster|
+        if ipvs_state.has_key?(cluster.fwmark.to_s)
+          cluster.resume_monitoring!
+
+          running_nodes = ipvs_state[cluster.fwmark.to_s]
+          cluster_nodes = cluster.nodes.map(&:address)
+
+          _remove_nodes(cluster, running_nodes - cluster_nodes)
+          _add_nodes(cluster, cluster_nodes - running_nodes)
+        end
+      end
+    end
+
+    def self._add_nodes(cluster, addresses)
+      addresses.each do |address|
+        BigBrother.logger.info "adding #{address} to cluster #{cluster}"
+        BigBrother.ipvs.start_node(cluster.fwmark, address, 100)
+      end
+    end
+
+    def self._remove_nodes(cluster, addresses)
+      addresses.each do |address|
+        BigBrother.logger.info "removing #{address} to cluster #{cluster}"
+        BigBrother.ipvs.stop_node(cluster.fwmark, address)
+      end
+    end
+
+    def self._parse_nodes(nodes)
+      nodes.map do |values|
+        Node.new(values['address'], values['port'], values['path'])
+      end
+    end
+
+  end
+end

data/lib/big_brother/ipvs.rb

@@ -0,0 +1,51 @@
+module BigBrother
+  class IPVS
+    def initialize(executor = ShellExecutor.new)
+      @executor = executor
+    end
+
+    def start_cluster(fwmark, scheduler)
+      @executor.invoke("ipvsadm --add-service --fwmark-service #{fwmark} --scheduler #{scheduler}")
+    end
+
+    def stop_cluster(fwmark)
+      @executor.invoke("ipvsadm --delete-service --fwmark-service #{fwmark}")
+    end
+
+    def edit_node(fwmark, address, weight)
+      @executor.invoke("ipvsadm --edit-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
+    end
+
+    def start_node(fwmark, address, weight)
+      @executor.invoke("ipvsadm --add-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
+    end
+
+    def stop_node(fwmark, address)
+      @executor.invoke("ipvsadm --delete-server --fwmark-service #{fwmark} --real-server #{address}")
+    end
+
+    def running_configuration
+      raw_output, status = @executor.invoke("ipvsadm --save --numeric")
+
+      parsed_lines = raw_output.split("\n").map do |line|
+        next if line =~ /-A/
+        {
+          :fwmark => line.slice(/-f (\d+)/, 1),
+          :real_server => line.slice(/-r ([0-9\.]+)/, 1)
+        }
+      end
+
+      _group_by_fwmark(parsed_lines.compact)
+    end
+
+
+    def _group_by_fwmark(parsed_lines)
+      parsed_lines.inject({}) do |accum, parsed_line|
+        accum[parsed_line[:fwmark]] ||= []
+        accum[parsed_line[:fwmark]] << parsed_line[:real_server]
+
+        accum
+      end
+    end
+  end
+end

data/lib/big_brother/logger.rb

@@ -0,0 +1,11 @@
+module BigBrother
+  class Logger
+    def write(message)
+      info(message)
+    end
+
+    def info(message)
+      EM.info(message)
+    end
+  end
+end

data/lib/big_brother/node.rb

@@ -0,0 +1,30 @@
+require 'net/http'
+
+module BigBrother
+  class Node
+    attr_reader :address, :port, :path
+
+    def initialize(address, port, path)
+      @address = address
+      @port = port
+      @path = path
+    end
+
+    def current_health
+      response = _get("http://#{@address}:#{@port}#{@path}")
+      _parse_health(response)
+    end
+
+    def _get(url)
+      EventMachine::HttpRequest.new(url).get
+    end
+
+    def _parse_health(http_response)
+      if http_response.response_header.has_key?('X_HEALTH')
+        http_response.response_header['X_HEALTH'].to_i
+      else
+        http_response.response.slice(/Health: (\d+)/, 1).to_i
+      end
+    end
+  end
+end

data/lib/big_brother/shell_executor.rb

@@ -0,0 +1,18 @@
+module BigBrother
+  class ShellExecutor
+    def invoke(command)
+      BigBrother.logger.info("Running command: #{command.inspect}")
+      _system(command)
+    end
+
+    def _system(command)
+      current_fiber = Fiber.current
+
+      EventMachine.system(command) do |output, status|
+        current_fiber.resume(output, status.exitstatus)
+      end
+
+      return Fiber.yield
+    end
+  end
+end

data/lib/big_brother/status_file.rb

@@ -0,0 +1,26 @@
+module BigBrother
+  class StatusFile
+    def initialize(*filenames)
+      @path = File.join(BigBrother.config_dir, *filenames)
+    end
+
+    def content
+      File.read(@path).chomp
+    end
+
+    def create(reason)
+      FileUtils.mkdir_p(File.dirname(@path))
+      File.open(@path, 'w') do |file|
+        file.puts(reason)
+      end
+    end
+
+    def delete
+      FileUtils.rm(@path)
+    end
+
+    def exists?
+      File.exists?(@path)
+    end
+  end
+end

data/lib/big_brother/ticker.rb

@@ -0,0 +1,28 @@
+module BigBrother
+  class Ticker
+
+    def self.pause(&block)
+      EM.cancel_timer(@timer)
+
+      while @outstanding_ticks > 0
+        EM::Synchrony.sleep(0.1)
+      end
+
+      block.call
+      schedule!
+    end
+
+    def self.schedule!
+      @outstanding_ticks = 0
+      @timer = EM::Synchrony.add_periodic_timer(0.1, &method(:tick))
+    end
+
+    def self.tick
+      @outstanding_ticks += 1
+      BigBrother.clusters.values.select(&:needs_check?).each do |cluster|
+        cluster.monitor_nodes
+      end
+      @outstanding_ticks -= 1
+    end
+  end
+end

data/lib/big_brother/version.rb

@@ -0,0 +1,3 @@
+module BigBrother
+  VERSION = "0.1.0"
+end

data/lib/sinatra/synchrony.rb

@@ -0,0 +1,40 @@
+# inlined from sinatra-synchrony
+require 'sinatra/base'
+require 'rack/fiber_pool'
+require 'eventmachine'
+require 'em-http-request'
+require 'em-synchrony'
+require 'em-resolv-replace'
+
+module Sinatra
+  module Synchrony
+    def self.registered(app)
+      app.disable :threaded
+    end
+
+    def setup_sessions(builder)
+      builder.use Rack::FiberPool, {:rescue_exception => handle_exception } unless test?
+      super
+    end
+
+    def handle_exception
+      Proc.new do |env, e|
+        if settings.show_exceptions?
+          request = Sinatra::Request.new(env)
+          printer = Sinatra::ShowExceptions.new(proc{ raise e })
+          s, h, b = printer.call(env)
+          [s, h, b]
+        else
+          [500, {}, ""]
+        end
+      end
+    end
+
+    class << self
+      def overload_tcpsocket!
+        require 'sinatra/synchrony/tcpsocket'
+      end
+    end
+  end
+  register Synchrony
+end

data/lib/thin/backends/tcp_server_with_callbacks.rb

@@ -0,0 +1,20 @@
+module Thin
+  module Backends
+    class TcpServerWithCallbacks < TcpServer
+      def initialize(host, port, options)
+        super(host, port)
+      end
+
+      def connect
+        super
+        Thin::Callbacks.after_connect_callbacks.each { |callback| callback.call }
+      end
+
+      def disconnect
+        Thin::Callbacks.before_disconnect_callbacks.each { |callback| callback.call }
+        super
+      end
+    end
+  end
+end
+

data/lib/thin/callback_rack_handler.rb

@@ -0,0 +1,14 @@
+module Thin
+  class CallbackRackHandler
+    def self.run(app, options)
+      server = ::Thin::Server.new(options[:Host] || '0.0.0.0',
+                                  options[:Port] || 8080,
+                                  app,
+                                  options)
+      yield server if block_given?
+      server.start
+    end
+  end
+end
+
+Rack::Handler.register 'thin-with-callbacks', Thin::CallbackRackHandler

data/lib/thin/callbacks.rb

@@ -0,0 +1,19 @@
+module Thin
+  class Callbacks
+    def self.after_connect_callbacks
+      @after_connect_callbacks ||= []
+    end
+
+    def self.after_connect(&block)
+      after_connect_callbacks << block
+    end
+
+    def self.before_disconnect_callbacks
+      @before_disconnect_callbacks ||= []
+    end
+
+    def self.before_disconnect(&block)
+      before_disconnect_callbacks << block
+    end
+  end
+end

data/spec/big_brother/app_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+
+module BigBrother
+  describe App do
+    def app
+      App
+    end
+
+    describe "/" do
+      it "returns the list of configured clusters and their status" do
+        BigBrother.clusters['one'] = Factory.cluster(:name => 'one', :fwmark => 1)
+        BigBrother.clusters['two'] = Factory.cluster(:name => 'two', :fwmark => 2)
+        BigBrother.clusters['three'] = Factory.cluster(:name => 'three', :fwmark => 3)
+        BigBrother.clusters['three'].start_monitoring!
+        BigBrother.clusters['four'] = Factory.cluster(:name => 'four', :fwmark => 4)
+
+        get "/"
+        last_response.status.should == 200
+        last_response.body.should include("one (1): not running")
+        last_response.body.should include("two (2): not running")
+        last_response.body.should include("three (3): running")
+        last_response.body.should include("four (4): not running")
+      end
+    end
+
+    describe "GET /cluster/:name" do
+      it "returns 'Running: false' when the cluster isn't running" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+
+        get "/cluster/test"
+
+        last_response.status.should == 200
+        last_response.body.should == "Running: false"
+      end
+
+      it "returns 'Running: true' when the cluster is running" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+
+        put "/cluster/test"
+        get "/cluster/test"
+
+        last_response.status.should == 200
+        last_response.body.should == "Running: true"
+      end
+
+      it "returns a 404 http status when the cluster is not found" do
+        get "/cluster/not_found"
+
+        last_response.status.should == 404
+      end
+    end
+
+    describe "PUT /cluster/:name" do
+      it "marks the cluster as monitored" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+
+        put "/cluster/test"
+
+        last_response.status.should == 200
+        last_response.body.should == ""
+        BigBrother.clusters['test'].should be_monitored
+      end
+
+      it "only starts monitoring the cluster once" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+
+        put "/cluster/test"
+        last_response.status.should == 200
+
+        put "/cluster/test"
+        last_response.status.should == 304
+      end
+
+      it "returns 'not found' if the cluster does not exist" do
+        put "/cluster/test"
+
+        last_response.status.should == 404
+        last_response.body.should == "Cluster test not found"
+      end
+
+      it "populates IPVS" do
+        first = Factory.node(:address => '127.0.0.1')
+        second = Factory.node(:address => '127.0.0.2')
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test', :fwmark => 100, :scheduler => 'wrr', :nodes => [first, second])
+
+        put "/cluster/test"
+
+        last_response.status.should == 200
+        last_response.body.should == ""
+        BigBrother.clusters['test'].should be_monitored
+        @recording_executor.commands.first.should == "ipvsadm --add-service --fwmark-service 100 --scheduler wrr"
+        @recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
+        @recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 100")
+      end
+    end
+
+    describe "DELETE /cluster/:name" do
+      it "marks the cluster as no longer monitored" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+        BigBrother.clusters['test'].start_monitoring!
+
+        delete "/cluster/test"
+
+        last_response.status.should == 200
+        last_response.body.should == ""
+        BigBrother.clusters['test'].should_not be_monitored
+      end
+
+      it "only stops monitoring the cluster once" do
+        BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
+
+        delete "/cluster/test"
+
+        last_response.status.should == 304
+        last_response.body.should == ""
+        BigBrother.clusters['test'].should_not be_monitored
+      end
+
+      it "returns 'not found' if the cluster does not exist" do
+        delete "/cluster/test"
+
+        last_response.status.should == 404
+        last_response.body.should == "Cluster test not found"
+      end
+    end
+  end
+end