big_brother 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +21 -0
- data/.rake_commit +1 -0
- data/.rvmrc +1 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +6 -0
- data/big_brother.gemspec +33 -0
- data/bin/bigbro +6 -0
- data/bin/ocf_big_brother +174 -0
- data/config.ru +5 -0
- data/lib/big_brother.rb +49 -0
- data/lib/big_brother/app.rb +30 -0
- data/lib/big_brother/cli.rb +82 -0
- data/lib/big_brother/cluster.rb +70 -0
- data/lib/big_brother/configuration.rb +48 -0
- data/lib/big_brother/ipvs.rb +51 -0
- data/lib/big_brother/logger.rb +11 -0
- data/lib/big_brother/node.rb +30 -0
- data/lib/big_brother/shell_executor.rb +18 -0
- data/lib/big_brother/status_file.rb +26 -0
- data/lib/big_brother/ticker.rb +28 -0
- data/lib/big_brother/version.rb +3 -0
- data/lib/sinatra/synchrony.rb +40 -0
- data/lib/thin/backends/tcp_server_with_callbacks.rb +20 -0
- data/lib/thin/callback_rack_handler.rb +14 -0
- data/lib/thin/callbacks.rb +19 -0
- data/spec/big_brother/app_spec.rb +127 -0
- data/spec/big_brother/cluster_spec.rb +102 -0
- data/spec/big_brother/configuration_spec.rb +81 -0
- data/spec/big_brother/ipvs_spec.rb +26 -0
- data/spec/big_brother/node_spec.rb +44 -0
- data/spec/big_brother/shell_executor_spec.rb +21 -0
- data/spec/big_brother/status_file_spec.rb +39 -0
- data/spec/big_brother/ticker_spec.rb +60 -0
- data/spec/big_brother/version_spec.rb +7 -0
- data/spec/big_brother_spec.rb +119 -0
- data/spec/spec_helper.rb +57 -0
- data/spec/support/example_config.yml +34 -0
- data/spec/support/factories/cluster_factory.rb +13 -0
- data/spec/support/factories/node_factory.rb +9 -0
- data/spec/support/ipvsadm +3 -0
- data/spec/support/mock_session.rb +14 -0
- data/spec/support/null_logger.rb +7 -0
- data/spec/support/playback_executor.rb +13 -0
- data/spec/support/recording_executor.rb +11 -0
- data/spec/support/stub_server.rb +22 -0
- metadata +271 -0
@@ -0,0 +1,70 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class Cluster
|
3
|
+
attr_reader :fwmark, :scheduler, :check_interval, :nodes
|
4
|
+
|
5
|
+
def initialize(name, attributes = {})
|
6
|
+
@name = name
|
7
|
+
@fwmark = attributes['fwmark']
|
8
|
+
@scheduler = attributes['scheduler']
|
9
|
+
@check_interval = attributes.fetch('check_interval', 1)
|
10
|
+
@monitored = false
|
11
|
+
@nodes = attributes.fetch('nodes', [])
|
12
|
+
@last_check = Time.new(0)
|
13
|
+
@up_file = BigBrother::StatusFile.new('up', @name)
|
14
|
+
@down_file = BigBrother::StatusFile.new('down', @name)
|
15
|
+
end
|
16
|
+
|
17
|
+
def monitored?
|
18
|
+
@monitored
|
19
|
+
end
|
20
|
+
|
21
|
+
def start_monitoring!
|
22
|
+
BigBrother.logger.info "starting monitoring on cluster #{to_s}"
|
23
|
+
BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
|
24
|
+
@nodes.each do |node|
|
25
|
+
BigBrother.ipvs.start_node(@fwmark, node.address, 100)
|
26
|
+
end
|
27
|
+
|
28
|
+
@monitored = true
|
29
|
+
end
|
30
|
+
|
31
|
+
def stop_monitoring!
|
32
|
+
BigBrother.logger.info "stopping monitoring on cluster #{to_s}"
|
33
|
+
BigBrother.ipvs.stop_cluster(@fwmark)
|
34
|
+
|
35
|
+
@monitored = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def resume_monitoring!
|
39
|
+
BigBrother.logger.info "resuming monitoring on cluster #{to_s}"
|
40
|
+
@monitored = true
|
41
|
+
end
|
42
|
+
|
43
|
+
def needs_check?
|
44
|
+
return false unless monitored?
|
45
|
+
@last_check + @check_interval < Time.now
|
46
|
+
end
|
47
|
+
|
48
|
+
def monitor_nodes
|
49
|
+
@nodes.each do |node|
|
50
|
+
BigBrother.ipvs.edit_node(@fwmark, node.address, _determine_weight(node))
|
51
|
+
end
|
52
|
+
|
53
|
+
@last_check = Time.now
|
54
|
+
end
|
55
|
+
|
56
|
+
def _determine_weight(node)
|
57
|
+
if @up_file.exists?
|
58
|
+
100
|
59
|
+
elsif @down_file.exists?
|
60
|
+
0
|
61
|
+
else
|
62
|
+
node.current_health
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def to_s
|
67
|
+
"#{@name} (#{@fwmark})"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class Configuration
|
3
|
+
def self.evaluate(file)
|
4
|
+
yaml = YAML.load(File.read(file))
|
5
|
+
assoc_array = yaml.map do |name, values|
|
6
|
+
nodes = _parse_nodes(values.delete('nodes'))
|
7
|
+
[name, Cluster.new(name, values.merge('nodes' => nodes))]
|
8
|
+
end
|
9
|
+
|
10
|
+
Hash[assoc_array]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.synchronize_with_ipvs(clusters, ipvs_state)
|
14
|
+
clusters.values.each do |cluster|
|
15
|
+
if ipvs_state.has_key?(cluster.fwmark.to_s)
|
16
|
+
cluster.resume_monitoring!
|
17
|
+
|
18
|
+
running_nodes = ipvs_state[cluster.fwmark.to_s]
|
19
|
+
cluster_nodes = cluster.nodes.map(&:address)
|
20
|
+
|
21
|
+
_remove_nodes(cluster, running_nodes - cluster_nodes)
|
22
|
+
_add_nodes(cluster, cluster_nodes - running_nodes)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self._add_nodes(cluster, addresses)
|
28
|
+
addresses.each do |address|
|
29
|
+
BigBrother.logger.info "adding #{address} to cluster #{cluster}"
|
30
|
+
BigBrother.ipvs.start_node(cluster.fwmark, address, 100)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def self._remove_nodes(cluster, addresses)
|
35
|
+
addresses.each do |address|
|
36
|
+
BigBrother.logger.info "removing #{address} to cluster #{cluster}"
|
37
|
+
BigBrother.ipvs.stop_node(cluster.fwmark, address)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self._parse_nodes(nodes)
|
42
|
+
nodes.map do |values|
|
43
|
+
Node.new(values['address'], values['port'], values['path'])
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class IPVS
|
3
|
+
def initialize(executor = ShellExecutor.new)
|
4
|
+
@executor = executor
|
5
|
+
end
|
6
|
+
|
7
|
+
def start_cluster(fwmark, scheduler)
|
8
|
+
@executor.invoke("ipvsadm --add-service --fwmark-service #{fwmark} --scheduler #{scheduler}")
|
9
|
+
end
|
10
|
+
|
11
|
+
def stop_cluster(fwmark)
|
12
|
+
@executor.invoke("ipvsadm --delete-service --fwmark-service #{fwmark}")
|
13
|
+
end
|
14
|
+
|
15
|
+
def edit_node(fwmark, address, weight)
|
16
|
+
@executor.invoke("ipvsadm --edit-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
|
17
|
+
end
|
18
|
+
|
19
|
+
def start_node(fwmark, address, weight)
|
20
|
+
@executor.invoke("ipvsadm --add-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
|
21
|
+
end
|
22
|
+
|
23
|
+
def stop_node(fwmark, address)
|
24
|
+
@executor.invoke("ipvsadm --delete-server --fwmark-service #{fwmark} --real-server #{address}")
|
25
|
+
end
|
26
|
+
|
27
|
+
def running_configuration
|
28
|
+
raw_output, status = @executor.invoke("ipvsadm --save --numeric")
|
29
|
+
|
30
|
+
parsed_lines = raw_output.split("\n").map do |line|
|
31
|
+
next if line =~ /-A/
|
32
|
+
{
|
33
|
+
:fwmark => line.slice(/-f (\d+)/, 1),
|
34
|
+
:real_server => line.slice(/-r ([0-9\.]+)/, 1)
|
35
|
+
}
|
36
|
+
end
|
37
|
+
|
38
|
+
_group_by_fwmark(parsed_lines.compact)
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
def _group_by_fwmark(parsed_lines)
|
43
|
+
parsed_lines.inject({}) do |accum, parsed_line|
|
44
|
+
accum[parsed_line[:fwmark]] ||= []
|
45
|
+
accum[parsed_line[:fwmark]] << parsed_line[:real_server]
|
46
|
+
|
47
|
+
accum
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
module BigBrother
|
4
|
+
class Node
|
5
|
+
attr_reader :address, :port, :path
|
6
|
+
|
7
|
+
def initialize(address, port, path)
|
8
|
+
@address = address
|
9
|
+
@port = port
|
10
|
+
@path = path
|
11
|
+
end
|
12
|
+
|
13
|
+
def current_health
|
14
|
+
response = _get("http://#{@address}:#{@port}#{@path}")
|
15
|
+
_parse_health(response)
|
16
|
+
end
|
17
|
+
|
18
|
+
def _get(url)
|
19
|
+
EventMachine::HttpRequest.new(url).get
|
20
|
+
end
|
21
|
+
|
22
|
+
def _parse_health(http_response)
|
23
|
+
if http_response.response_header.has_key?('X_HEALTH')
|
24
|
+
http_response.response_header['X_HEALTH'].to_i
|
25
|
+
else
|
26
|
+
http_response.response.slice(/Health: (\d+)/, 1).to_i
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class ShellExecutor
|
3
|
+
def invoke(command)
|
4
|
+
BigBrother.logger.info("Running command: #{command.inspect}")
|
5
|
+
_system(command)
|
6
|
+
end
|
7
|
+
|
8
|
+
def _system(command)
|
9
|
+
current_fiber = Fiber.current
|
10
|
+
|
11
|
+
EventMachine.system(command) do |output, status|
|
12
|
+
current_fiber.resume(output, status.exitstatus)
|
13
|
+
end
|
14
|
+
|
15
|
+
return Fiber.yield
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class StatusFile
|
3
|
+
def initialize(*filenames)
|
4
|
+
@path = File.join(BigBrother.config_dir, *filenames)
|
5
|
+
end
|
6
|
+
|
7
|
+
def content
|
8
|
+
File.read(@path).chomp
|
9
|
+
end
|
10
|
+
|
11
|
+
def create(reason)
|
12
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
13
|
+
File.open(@path, 'w') do |file|
|
14
|
+
file.puts(reason)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def delete
|
19
|
+
FileUtils.rm(@path)
|
20
|
+
end
|
21
|
+
|
22
|
+
def exists?
|
23
|
+
File.exists?(@path)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class Ticker
|
3
|
+
|
4
|
+
def self.pause(&block)
|
5
|
+
EM.cancel_timer(@timer)
|
6
|
+
|
7
|
+
while @outstanding_ticks > 0
|
8
|
+
EM::Synchrony.sleep(0.1)
|
9
|
+
end
|
10
|
+
|
11
|
+
block.call
|
12
|
+
schedule!
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.schedule!
|
16
|
+
@outstanding_ticks = 0
|
17
|
+
@timer = EM::Synchrony.add_periodic_timer(0.1, &method(:tick))
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.tick
|
21
|
+
@outstanding_ticks += 1
|
22
|
+
BigBrother.clusters.values.select(&:needs_check?).each do |cluster|
|
23
|
+
cluster.monitor_nodes
|
24
|
+
end
|
25
|
+
@outstanding_ticks -= 1
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# inlined from sinatra-synchrony
|
2
|
+
require 'sinatra/base'
|
3
|
+
require 'rack/fiber_pool'
|
4
|
+
require 'eventmachine'
|
5
|
+
require 'em-http-request'
|
6
|
+
require 'em-synchrony'
|
7
|
+
require 'em-resolv-replace'
|
8
|
+
|
9
|
+
module Sinatra
|
10
|
+
module Synchrony
|
11
|
+
def self.registered(app)
|
12
|
+
app.disable :threaded
|
13
|
+
end
|
14
|
+
|
15
|
+
def setup_sessions(builder)
|
16
|
+
builder.use Rack::FiberPool, {:rescue_exception => handle_exception } unless test?
|
17
|
+
super
|
18
|
+
end
|
19
|
+
|
20
|
+
def handle_exception
|
21
|
+
Proc.new do |env, e|
|
22
|
+
if settings.show_exceptions?
|
23
|
+
request = Sinatra::Request.new(env)
|
24
|
+
printer = Sinatra::ShowExceptions.new(proc{ raise e })
|
25
|
+
s, h, b = printer.call(env)
|
26
|
+
[s, h, b]
|
27
|
+
else
|
28
|
+
[500, {}, ""]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class << self
|
34
|
+
def overload_tcpsocket!
|
35
|
+
require 'sinatra/synchrony/tcpsocket'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
register Synchrony
|
40
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Thin
|
2
|
+
module Backends
|
3
|
+
class TcpServerWithCallbacks < TcpServer
|
4
|
+
def initialize(host, port, options)
|
5
|
+
super(host, port)
|
6
|
+
end
|
7
|
+
|
8
|
+
def connect
|
9
|
+
super
|
10
|
+
Thin::Callbacks.after_connect_callbacks.each { |callback| callback.call }
|
11
|
+
end
|
12
|
+
|
13
|
+
def disconnect
|
14
|
+
Thin::Callbacks.before_disconnect_callbacks.each { |callback| callback.call }
|
15
|
+
super
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Thin
|
2
|
+
class CallbackRackHandler
|
3
|
+
def self.run(app, options)
|
4
|
+
server = ::Thin::Server.new(options[:Host] || '0.0.0.0',
|
5
|
+
options[:Port] || 8080,
|
6
|
+
app,
|
7
|
+
options)
|
8
|
+
yield server if block_given?
|
9
|
+
server.start
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
Rack::Handler.register 'thin-with-callbacks', Thin::CallbackRackHandler
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Thin
|
2
|
+
class Callbacks
|
3
|
+
def self.after_connect_callbacks
|
4
|
+
@after_connect_callbacks ||= []
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.after_connect(&block)
|
8
|
+
after_connect_callbacks << block
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.before_disconnect_callbacks
|
12
|
+
@before_disconnect_callbacks ||= []
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.before_disconnect(&block)
|
16
|
+
before_disconnect_callbacks << block
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module BigBrother
|
4
|
+
describe App do
|
5
|
+
def app
|
6
|
+
App
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "/" do
|
10
|
+
it "returns the list of configured clusters and their status" do
|
11
|
+
BigBrother.clusters['one'] = Factory.cluster(:name => 'one', :fwmark => 1)
|
12
|
+
BigBrother.clusters['two'] = Factory.cluster(:name => 'two', :fwmark => 2)
|
13
|
+
BigBrother.clusters['three'] = Factory.cluster(:name => 'three', :fwmark => 3)
|
14
|
+
BigBrother.clusters['three'].start_monitoring!
|
15
|
+
BigBrother.clusters['four'] = Factory.cluster(:name => 'four', :fwmark => 4)
|
16
|
+
|
17
|
+
get "/"
|
18
|
+
last_response.status.should == 200
|
19
|
+
last_response.body.should include("one (1): not running")
|
20
|
+
last_response.body.should include("two (2): not running")
|
21
|
+
last_response.body.should include("three (3): running")
|
22
|
+
last_response.body.should include("four (4): not running")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "GET /cluster/:name" do
|
27
|
+
it "returns 'Running: false' when the cluster isn't running" do
|
28
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
29
|
+
|
30
|
+
get "/cluster/test"
|
31
|
+
|
32
|
+
last_response.status.should == 200
|
33
|
+
last_response.body.should == "Running: false"
|
34
|
+
end
|
35
|
+
|
36
|
+
it "returns 'Running: true' when the cluster is running" do
|
37
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
38
|
+
|
39
|
+
put "/cluster/test"
|
40
|
+
get "/cluster/test"
|
41
|
+
|
42
|
+
last_response.status.should == 200
|
43
|
+
last_response.body.should == "Running: true"
|
44
|
+
end
|
45
|
+
|
46
|
+
it "returns a 404 http status when the cluster is not found" do
|
47
|
+
get "/cluster/not_found"
|
48
|
+
|
49
|
+
last_response.status.should == 404
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe "PUT /cluster/:name" do
|
54
|
+
it "marks the cluster as monitored" do
|
55
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
56
|
+
|
57
|
+
put "/cluster/test"
|
58
|
+
|
59
|
+
last_response.status.should == 200
|
60
|
+
last_response.body.should == ""
|
61
|
+
BigBrother.clusters['test'].should be_monitored
|
62
|
+
end
|
63
|
+
|
64
|
+
it "only starts monitoring the cluster once" do
|
65
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
66
|
+
|
67
|
+
put "/cluster/test"
|
68
|
+
last_response.status.should == 200
|
69
|
+
|
70
|
+
put "/cluster/test"
|
71
|
+
last_response.status.should == 304
|
72
|
+
end
|
73
|
+
|
74
|
+
it "returns 'not found' if the cluster does not exist" do
|
75
|
+
put "/cluster/test"
|
76
|
+
|
77
|
+
last_response.status.should == 404
|
78
|
+
last_response.body.should == "Cluster test not found"
|
79
|
+
end
|
80
|
+
|
81
|
+
it "populates IPVS" do
|
82
|
+
first = Factory.node(:address => '127.0.0.1')
|
83
|
+
second = Factory.node(:address => '127.0.0.2')
|
84
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test', :fwmark => 100, :scheduler => 'wrr', :nodes => [first, second])
|
85
|
+
|
86
|
+
put "/cluster/test"
|
87
|
+
|
88
|
+
last_response.status.should == 200
|
89
|
+
last_response.body.should == ""
|
90
|
+
BigBrother.clusters['test'].should be_monitored
|
91
|
+
@recording_executor.commands.first.should == "ipvsadm --add-service --fwmark-service 100 --scheduler wrr"
|
92
|
+
@recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
|
93
|
+
@recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 100")
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
describe "DELETE /cluster/:name" do
|
98
|
+
it "marks the cluster as no longer monitored" do
|
99
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
100
|
+
BigBrother.clusters['test'].start_monitoring!
|
101
|
+
|
102
|
+
delete "/cluster/test"
|
103
|
+
|
104
|
+
last_response.status.should == 200
|
105
|
+
last_response.body.should == ""
|
106
|
+
BigBrother.clusters['test'].should_not be_monitored
|
107
|
+
end
|
108
|
+
|
109
|
+
it "only stops monitoring the cluster once" do
|
110
|
+
BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
|
111
|
+
|
112
|
+
delete "/cluster/test"
|
113
|
+
|
114
|
+
last_response.status.should == 304
|
115
|
+
last_response.body.should == ""
|
116
|
+
BigBrother.clusters['test'].should_not be_monitored
|
117
|
+
end
|
118
|
+
|
119
|
+
it "returns 'not found' if the cluster does not exist" do
|
120
|
+
delete "/cluster/test"
|
121
|
+
|
122
|
+
last_response.status.should == 404
|
123
|
+
last_response.body.should == "Cluster test not found"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|