big_brother 0.1.0

data/spec/big_brother/cluster_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+describe BigBrother::Cluster do
+  describe "#start_monitoring!" do
+    it "marks the cluster as monitored" do
+      cluster = Factory.cluster
+      cluster.should_not be_monitored
+      cluster.start_monitoring!
+      cluster.should be_monitored
+    end
+
+    it "starts the service in IPVS" do
+      cluster = Factory.cluster(:fwmark => 100, :scheduler => 'wrr')
+
+      cluster.start_monitoring!
+      @recording_executor.commands.should include('ipvsadm --add-service --fwmark-service 100 --scheduler wrr')
+    end
+  end
+
+  describe "#stop_monitoring!" do
+    it "marks the cluster as unmonitored" do
+      cluster = Factory.cluster(:fwmark => 100)
+
+      cluster.start_monitoring!
+      cluster.should be_monitored
+
+      cluster.stop_monitoring!
+      cluster.should_not be_monitored
+      @recording_executor.commands.should include("ipvsadm --delete-service --fwmark-service 100")
+    end
+  end
+
+  describe "#needs_check?" do
+    it "requires the cluster to be monitored" do
+      cluster = Factory.cluster
+      cluster.needs_check?.should be_false
+      cluster.start_monitoring!
+      cluster.needs_check?.should be_true
+    end
+  end
+
+  describe "#monitor_nodes" do
+    it "marks the cluster as no longer requiring monitoring" do
+      cluster = Factory.cluster
+      cluster.start_monitoring!
+      cluster.needs_check?.should be_true
+      cluster.monitor_nodes
+      cluster.needs_check?.should be_false
+    end
+
+    it "updates the weight for each node" do
+      node = Factory.node(:address => '127.0.0.1')
+      node.should_receive(:current_health).and_return(56)
+      cluster = Factory.cluster(:fwmark => 100, :nodes => [node])
+
+      cluster.monitor_nodes
+
+      @recording_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
+    end
+
+    it "sets the weight to 100 for each node if an upfile exists" do
+      node = Factory.node(:address => '127.0.0.1')
+      node.stub(:current_health).and_return(56)
+      cluster = Factory.cluster(:name => 'test', :fwmark => 100, :nodes => [node])
+
+      BigBrother::StatusFile.new('up', 'test').create('Up for testing')
+
+      cluster.monitor_nodes
+
+      @recording_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
+    end
+
+    it "sets the weight to 0 for each node if a downfile exists" do
+      node = Factory.node(:address => '127.0.0.1')
+      node.stub(:current_health).and_return(56)
+      cluster = Factory.cluster(:name => 'test', :fwmark => 100, :nodes => [node])
+
+      BigBrother::StatusFile.new('down', 'test').create('Down for testing')
+
+      cluster.monitor_nodes
+
+      @recording_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 0")
+    end
+  end
+
+  describe "#resume_monitoring!" do
+    it "marks the cluster as monitored" do
+      cluster = Factory.cluster
+
+      cluster.monitored?.should be_false
+      cluster.resume_monitoring!
+      cluster.monitored?.should be_true
+    end
+  end
+
+  describe "#to_s" do
+    it "is the clusters name and fwmark" do
+      cluster = Factory.cluster(:name => 'name', :fwmark => 100)
+      cluster.to_s.should == "name (100)"
+    end
+  end
+end

data/spec/big_brother/configuration_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe BigBrother::Configuration do
+  describe '.evaluate' do
+    it 'returns a hash of clusters' do
+      clusters = BigBrother::Configuration.evaluate(TEST_CONFIG)
+
+      clusters['test1'].check_interval.should == 1
+      clusters['test1'].scheduler.should == 'wrr'
+      clusters['test1'].fwmark.should == 1
+
+      clusters['test2'].check_interval.should == 1
+      clusters['test2'].scheduler.should == 'wrr'
+      clusters['test2'].fwmark.should == 2
+
+      clusters['test3'].check_interval.should == 1
+      clusters['test3'].scheduler.should == 'wrr'
+      clusters['test3'].fwmark.should == 3
+    end
+
+    it 'populates a clusters nodes' do
+      clusters = BigBrother::Configuration.evaluate(TEST_CONFIG)
+
+      clusters['test1'].nodes.length.should == 2
+
+      clusters['test1'].nodes[0].address == '127.0.0.1'
+      clusters['test1'].nodes[0].port == '9001'
+      clusters['test1'].nodes[0].path == '/test/valid'
+
+      clusters['test1'].nodes[1].address == '127.0.0.1'
+      clusters['test1'].nodes[1].port == '9002'
+      clusters['test1'].nodes[1].path == '/test/valid'
+    end
+  end
+
+  describe '.synchronize_with_ipvs' do
+    it "monitors clusters that were already monitored" do
+      clusters = {}
+      clusters['one'] = Factory.cluster(:fwmark => 1)
+      clusters['two'] = Factory.cluster(:fwmark => 2)
+      clusters['three'] = Factory.cluster(:fwmark => 3)
+
+      ipvs_state = {}
+      ipvs_state['1'] = ['127.0.0.1']
+      ipvs_state['3'] = ['127.0.0.1']
+
+      BigBrother::Configuration.synchronize_with_ipvs(clusters, ipvs_state)
+
+      clusters['one'].should be_monitored
+      clusters['two'].should_not be_monitored
+      clusters['three'].should be_monitored
+    end
+
+    it "does not attempt to re-add the services it was monitoring" do
+      clusters = { 'one' => Factory.cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')]) }
+      ipvs_state = { '1' => ['127.0.0.1'] }
+
+      BigBrother::Configuration.synchronize_with_ipvs(clusters, ipvs_state)
+
+      @recording_executor.commands.should be_empty
+    end
+
+    it "removes nodes that are no longer part of the cluster" do
+      clusters = { 'one' => Factory.cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')]) }
+      ipvs_state = { '1' => ['127.0.1.1', '127.0.0.1'] }
+
+      BigBrother::Configuration.synchronize_with_ipvs(clusters, ipvs_state)
+
+      @recording_executor.commands.last.should == "ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1"
+    end
+
+    it "adds new nodes to the cluster" do
+      clusters = { 'one' => Factory.cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1")]) }
+      ipvs_state = { '1' => ['127.0.0.1'] }
+
+      BigBrother::Configuration.synchronize_with_ipvs(clusters, ipvs_state)
+
+      @recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 100")
+    end
+  end
+end

data/spec/big_brother/ipvs_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe BigBrother::IPVS do
+  describe "#running_configuration" do
+    it "returns a parsed version of the running config" do
+      playback = PlaybackExecutor.new
+      playback.add_response(<<-OUTPUT, 0)
+-A -f 3 -s wrr
+-a -f 3 -r 10.0.1.220:80 -i -w 1
+-a -f 3 -r 10.0.1.221:80 -i -w 1
+-a -f 3 -r 10.0.1.222:80 -i -w 1
+-A -f 1 -s wrr
+-a -f 1 -r 10.0.1.223:80 -i -w 1
+-a -f 1 -r 10.0.1.224:80 -i -w 1
+-A -f 2 -s wrr
+-a -f 2 -r 10.0.1.225:80 -i -w 1
+      OUTPUT
+      config = BigBrother::IPVS.new(playback).running_configuration
+
+      config.size.should == 3
+      config['3'].should == ['10.0.1.220', '10.0.1.221', '10.0.1.222']
+      config['1'].should == ['10.0.1.223', '10.0.1.224']
+      config['2'].should == ['10.0.1.225']
+    end
+  end
+end

data/spec/big_brother/node_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe BigBrother::Node do
+  describe "#current_health" do
+    run_in_reactor
+
+    it "returns its health" do
+      StubServer.new(<<-HTTP, 0.25)
+HTTP/1.0 200 OK
+Connection: close
+
+Health: 59
+HTTP
+      node = BigBrother::Node.new("127.0.0.1", 8081, "/test/status")
+      node.current_health.should == 59
+    end
+
+    it "returns 0 for an unknown service" do
+      StubServer.new(<<-HTTP, 0.25)
+HTTP/1.0 503 Service Unavailable
+Connection: close
+HTTP
+      node = BigBrother::Node.new("127.0.0.1", 8081, "/test/status")
+      node.current_health.should == 0
+    end
+
+    it "returns 0 for an unknown DNS entry" do
+      node = BigBrother::Node.new("junk.local", 8081, "/test/status")
+      node.current_health.should == 0
+    end
+
+    it "returns the health if it is passed in a header" do
+      StubServer.new(<<-HTTP, 0.25)
+HTTP/1.0 200 OK
+Connection: close
+X-Health: 61
+
+This part is for people.
+HTTP
+      node = BigBrother::Node.new("127.0.0.1", 8081, "/test/status")
+      node.current_health.should == 61
+    end
+  end
+end

data/spec/big_brother/shell_executor_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe BigBrother::ShellExecutor do
+  describe "#invoke" do
+    run_in_reactor
+
+    it "runs a command" do
+      executor = BigBrother::ShellExecutor.new
+      output, exit_status = executor.invoke('echo hi')
+      output.should == "hi\n"
+      exit_status.should == 0
+    end
+
+    it "failed command" do
+      executor = BigBrother::ShellExecutor.new
+      output, exit_status = executor.invoke('test -e /this/isnt/here')
+      output.should == ""
+      exit_status.should == 1
+    end
+  end
+end

data/spec/big_brother/status_file_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe BigBrother::StatusFile do
+  describe "create" do
+    it "creates a nested file" do
+      status_file = BigBrother::StatusFile.new("foo", "bar")
+      status_file.create("for testing")
+
+      status_file.exists?.should == true
+    end
+
+    it "creates a file" do
+      status_file = BigBrother::StatusFile.new("foo")
+      status_file.create("for testing")
+
+      status_file.exists?.should == true
+    end
+
+    it "writes the content" do
+      status_file = BigBrother::StatusFile.new("foo")
+      status_file.create("for testing")
+
+      status_file.content.should == "for testing"
+    end
+  end
+
+  describe "delete" do
+    it "removes the file" do
+      status_file = BigBrother::StatusFile.new("foo")
+      status_file.create("for testing")
+
+      status_file.exists?.should be_true
+
+      status_file.delete
+
+      status_file.exists?.should be_false
+    end
+  end
+end

data/spec/big_brother/ticker_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe BigBrother::Ticker do
+  describe ".schedule!" do
+    it "schedules the ticker to run ten times every second" do
+      EventMachine.should_receive(:add_periodic_timer).with(0.1)
+
+      BigBrother::Ticker.schedule!
+    end
+  end
+
+  describe ".tick" do
+    describe "end-to-end" do
+      run_in_reactor
+      with_litmus_server '127.0.0.1', 8081, 74
+      with_litmus_server public_ip_address, 8082, 76
+
+      it "monitors clusters requiring monitoring" do
+        BigBrother.clusters['test'] = Factory.cluster(
+          :fwmark => 100,
+          :nodes => [
+            Factory.node(:address  => '127.0.0.1', :port => 8081),
+            Factory.node(:address  => public_ip_address, :port => 8082)
+          ]
+        )
+        BigBrother.clusters['test'].start_monitoring!
+        @recording_executor.commands.clear
+
+        BigBrother::Ticker.tick
+
+        @recording_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 74")
+        @recording_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server #{public_ip_address} --ipip --weight 76")
+      end
+
+      it "only monitors a cluster once in the given interval" do
+        BigBrother.clusters['test'] = Factory.cluster(
+          :fwmark => 100,
+          :nodes => [Factory.node(:address  => '127.0.0.1', :port => 8081)]
+        )
+        BigBrother.clusters['test'].start_monitoring!
+        @recording_executor.commands.clear
+
+        BigBrother::Ticker.tick
+        BigBrother::Ticker.tick
+
+        @recording_executor.commands.should == ["ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 74"]
+      end
+    end
+
+    it "monitors clusters requiring monitoring" do
+      BigBrother.clusters['one'] = Factory.cluster
+      BigBrother.clusters['two'] = Factory.cluster
+      BigBrother.clusters['two'].start_monitoring!
+
+      BigBrother.clusters['two'].should_receive(:monitor_nodes)
+
+      BigBrother::Ticker.tick
+    end
+  end
+end

data/spec/big_brother/version_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe BigBrother::VERSION do
+  it "is not nil" do
+    BigBrother::VERSION.should_not be_nil
+  end
+end

data/spec/big_brother_spec.rb

@@ -0,0 +1,119 @@
+require 'spec_helper'
+
+describe BigBrother do
+  describe '.configure' do
+    it "reads the configuration file" do
+      BigBrother.configure(TEST_CONFIG)
+      BigBrother.clusters.size.should == 3
+    end
+
+    it "synchronizes the configuration with the current state of IPVS" do
+      playback = PlaybackExecutor.new
+      playback.add_response(<<-OUTPUT, 0)
+-A -f 1 -s wrr
+-a -f 1 -r 10.0.1.223:80 -i -w 1
+-a -f 1 -r 10.0.1.224:80 -i -w 1
+-A -f 2 -s wrr
+-a -f 2 -r 10.0.1.225:80 -i -w 1
+      OUTPUT
+      BigBrother.ipvs = BigBrother::IPVS.new(playback)
+      BigBrother.configure(TEST_CONFIG)
+
+      BigBrother.clusters['test1'].should be_monitored
+      BigBrother.clusters['test2'].should be_monitored
+    end
+  end
+
+  describe '.reconfigure' do
+    run_in_reactor
+    around(:each) do |spec|
+      response_time = 1 #seconds
+      server = StubServer.new(<<HTTP, response_time, 9001, '127.0.0.1')
+HTTP/1.0 200 OK
+Connection: close
+
+Health: 50
+HTTP
+      spec.run
+      server.stop
+    end
+    it "reconfigures the clusters" do
+      config_file = Tempfile.new('config.yml')
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  checkInterval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/valid
+EOF
+      end
+      BigBrother.configure(config_file)
+      BigBrother.start_ticker!
+      BigBrother.clusters['test1'].nodes.first.path.should == "/test/valid"
+
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  checkInterval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/another/path
+EOF
+      end
+      BigBrother.reconfigure
+      BigBrother.clusters['test1'].nodes.first.path.should == "/test/another/path"
+    end
+
+    it "stops the ticker and reconfigures after it has finished all its ticks" do
+      config_file = Tempfile.new('config.yml')
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  checkInterval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/valid
+EOF
+      end
+      BigBrother.configure(config_file)
+      BigBrother.clusters['test1'].start_monitoring!
+      @recording_executor.commands.clear
+
+      BigBrother.start_ticker!
+
+      EM::Synchrony.sleep(0.2)
+
+      File.open(config_file, 'w') do |f|
+        f.puts(<<-EOF)
+---
+test1:
+  checkInterval: 1
+  scheduler: wrr
+  fwmark: 1
+  nodes:
+  - address: 127.0.0.1
+    port: 9001
+    path: /test/another/path
+EOF
+      end
+      BigBrother.reconfigure
+      BigBrother.clusters['test1'].nodes.first.path.should == "/test/another/path"
+
+      @recording_executor.commands.first.should include("--weight 50")
+      @recording_executor.commands.last.should == "ipvsadm --save --numeric"
+    end
+  end
+end