big_brother 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/.gitignore +21 -0
  2. data/.rake_commit +1 -0
  3. data/.rvmrc +1 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE +22 -0
  6. data/README.md +29 -0
  7. data/Rakefile +6 -0
  8. data/big_brother.gemspec +33 -0
  9. data/bin/bigbro +6 -0
  10. data/bin/ocf_big_brother +174 -0
  11. data/config.ru +5 -0
  12. data/lib/big_brother.rb +49 -0
  13. data/lib/big_brother/app.rb +30 -0
  14. data/lib/big_brother/cli.rb +82 -0
  15. data/lib/big_brother/cluster.rb +70 -0
  16. data/lib/big_brother/configuration.rb +48 -0
  17. data/lib/big_brother/ipvs.rb +51 -0
  18. data/lib/big_brother/logger.rb +11 -0
  19. data/lib/big_brother/node.rb +30 -0
  20. data/lib/big_brother/shell_executor.rb +18 -0
  21. data/lib/big_brother/status_file.rb +26 -0
  22. data/lib/big_brother/ticker.rb +28 -0
  23. data/lib/big_brother/version.rb +3 -0
  24. data/lib/sinatra/synchrony.rb +40 -0
  25. data/lib/thin/backends/tcp_server_with_callbacks.rb +20 -0
  26. data/lib/thin/callback_rack_handler.rb +14 -0
  27. data/lib/thin/callbacks.rb +19 -0
  28. data/spec/big_brother/app_spec.rb +127 -0
  29. data/spec/big_brother/cluster_spec.rb +102 -0
  30. data/spec/big_brother/configuration_spec.rb +81 -0
  31. data/spec/big_brother/ipvs_spec.rb +26 -0
  32. data/spec/big_brother/node_spec.rb +44 -0
  33. data/spec/big_brother/shell_executor_spec.rb +21 -0
  34. data/spec/big_brother/status_file_spec.rb +39 -0
  35. data/spec/big_brother/ticker_spec.rb +60 -0
  36. data/spec/big_brother/version_spec.rb +7 -0
  37. data/spec/big_brother_spec.rb +119 -0
  38. data/spec/spec_helper.rb +57 -0
  39. data/spec/support/example_config.yml +34 -0
  40. data/spec/support/factories/cluster_factory.rb +13 -0
  41. data/spec/support/factories/node_factory.rb +9 -0
  42. data/spec/support/ipvsadm +3 -0
  43. data/spec/support/mock_session.rb +14 -0
  44. data/spec/support/null_logger.rb +7 -0
  45. data/spec/support/playback_executor.rb +13 -0
  46. data/spec/support/recording_executor.rb +11 -0
  47. data/spec/support/stub_server.rb +22 -0
  48. metadata +271 -0
@@ -0,0 +1,70 @@
1
+ module BigBrother
2
+ class Cluster
3
+ attr_reader :fwmark, :scheduler, :check_interval, :nodes
4
+
5
+ def initialize(name, attributes = {})
6
+ @name = name
7
+ @fwmark = attributes['fwmark']
8
+ @scheduler = attributes['scheduler']
9
+ @check_interval = attributes.fetch('check_interval', 1)
10
+ @monitored = false
11
+ @nodes = attributes.fetch('nodes', [])
12
+ @last_check = Time.new(0)
13
+ @up_file = BigBrother::StatusFile.new('up', @name)
14
+ @down_file = BigBrother::StatusFile.new('down', @name)
15
+ end
16
+
17
+ def monitored?
18
+ @monitored
19
+ end
20
+
21
+ def start_monitoring!
22
+ BigBrother.logger.info "starting monitoring on cluster #{to_s}"
23
+ BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
24
+ @nodes.each do |node|
25
+ BigBrother.ipvs.start_node(@fwmark, node.address, 100)
26
+ end
27
+
28
+ @monitored = true
29
+ end
30
+
31
+ def stop_monitoring!
32
+ BigBrother.logger.info "stopping monitoring on cluster #{to_s}"
33
+ BigBrother.ipvs.stop_cluster(@fwmark)
34
+
35
+ @monitored = false
36
+ end
37
+
38
+ def resume_monitoring!
39
+ BigBrother.logger.info "resuming monitoring on cluster #{to_s}"
40
+ @monitored = true
41
+ end
42
+
43
+ def needs_check?
44
+ return false unless monitored?
45
+ @last_check + @check_interval < Time.now
46
+ end
47
+
48
+ def monitor_nodes
49
+ @nodes.each do |node|
50
+ BigBrother.ipvs.edit_node(@fwmark, node.address, _determine_weight(node))
51
+ end
52
+
53
+ @last_check = Time.now
54
+ end
55
+
56
+ def _determine_weight(node)
57
+ if @up_file.exists?
58
+ 100
59
+ elsif @down_file.exists?
60
+ 0
61
+ else
62
+ node.current_health
63
+ end
64
+ end
65
+
66
+ def to_s
67
+ "#{@name} (#{@fwmark})"
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,48 @@
1
+ module BigBrother
2
+ class Configuration
3
+ def self.evaluate(file)
4
+ yaml = YAML.load(File.read(file))
5
+ assoc_array = yaml.map do |name, values|
6
+ nodes = _parse_nodes(values.delete('nodes'))
7
+ [name, Cluster.new(name, values.merge('nodes' => nodes))]
8
+ end
9
+
10
+ Hash[assoc_array]
11
+ end
12
+
13
+ def self.synchronize_with_ipvs(clusters, ipvs_state)
14
+ clusters.values.each do |cluster|
15
+ if ipvs_state.has_key?(cluster.fwmark.to_s)
16
+ cluster.resume_monitoring!
17
+
18
+ running_nodes = ipvs_state[cluster.fwmark.to_s]
19
+ cluster_nodes = cluster.nodes.map(&:address)
20
+
21
+ _remove_nodes(cluster, running_nodes - cluster_nodes)
22
+ _add_nodes(cluster, cluster_nodes - running_nodes)
23
+ end
24
+ end
25
+ end
26
+
27
+ def self._add_nodes(cluster, addresses)
28
+ addresses.each do |address|
29
+ BigBrother.logger.info "adding #{address} to cluster #{cluster}"
30
+ BigBrother.ipvs.start_node(cluster.fwmark, address, 100)
31
+ end
32
+ end
33
+
34
+ def self._remove_nodes(cluster, addresses)
35
+ addresses.each do |address|
36
+ BigBrother.logger.info "removing #{address} to cluster #{cluster}"
37
+ BigBrother.ipvs.stop_node(cluster.fwmark, address)
38
+ end
39
+ end
40
+
41
+ def self._parse_nodes(nodes)
42
+ nodes.map do |values|
43
+ Node.new(values['address'], values['port'], values['path'])
44
+ end
45
+ end
46
+
47
+ end
48
+ end
@@ -0,0 +1,51 @@
1
+ module BigBrother
2
+ class IPVS
3
+ def initialize(executor = ShellExecutor.new)
4
+ @executor = executor
5
+ end
6
+
7
+ def start_cluster(fwmark, scheduler)
8
+ @executor.invoke("ipvsadm --add-service --fwmark-service #{fwmark} --scheduler #{scheduler}")
9
+ end
10
+
11
+ def stop_cluster(fwmark)
12
+ @executor.invoke("ipvsadm --delete-service --fwmark-service #{fwmark}")
13
+ end
14
+
15
+ def edit_node(fwmark, address, weight)
16
+ @executor.invoke("ipvsadm --edit-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
17
+ end
18
+
19
+ def start_node(fwmark, address, weight)
20
+ @executor.invoke("ipvsadm --add-server --fwmark-service #{fwmark} --real-server #{address} --ipip --weight #{weight}")
21
+ end
22
+
23
+ def stop_node(fwmark, address)
24
+ @executor.invoke("ipvsadm --delete-server --fwmark-service #{fwmark} --real-server #{address}")
25
+ end
26
+
27
+ def running_configuration
28
+ raw_output, status = @executor.invoke("ipvsadm --save --numeric")
29
+
30
+ parsed_lines = raw_output.split("\n").map do |line|
31
+ next if line =~ /-A/
32
+ {
33
+ :fwmark => line.slice(/-f (\d+)/, 1),
34
+ :real_server => line.slice(/-r ([0-9\.]+)/, 1)
35
+ }
36
+ end
37
+
38
+ _group_by_fwmark(parsed_lines.compact)
39
+ end
40
+
41
+
42
+ def _group_by_fwmark(parsed_lines)
43
+ parsed_lines.inject({}) do |accum, parsed_line|
44
+ accum[parsed_line[:fwmark]] ||= []
45
+ accum[parsed_line[:fwmark]] << parsed_line[:real_server]
46
+
47
+ accum
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,11 @@
1
+ module BigBrother
2
+ class Logger
3
+ def write(message)
4
+ info(message)
5
+ end
6
+
7
+ def info(message)
8
+ EM.info(message)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,30 @@
1
+ require 'net/http'
2
+
3
+ module BigBrother
4
+ class Node
5
+ attr_reader :address, :port, :path
6
+
7
+ def initialize(address, port, path)
8
+ @address = address
9
+ @port = port
10
+ @path = path
11
+ end
12
+
13
+ def current_health
14
+ response = _get("http://#{@address}:#{@port}#{@path}")
15
+ _parse_health(response)
16
+ end
17
+
18
+ def _get(url)
19
+ EventMachine::HttpRequest.new(url).get
20
+ end
21
+
22
+ def _parse_health(http_response)
23
+ if http_response.response_header.has_key?('X_HEALTH')
24
+ http_response.response_header['X_HEALTH'].to_i
25
+ else
26
+ http_response.response.slice(/Health: (\d+)/, 1).to_i
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,18 @@
1
+ module BigBrother
2
+ class ShellExecutor
3
+ def invoke(command)
4
+ BigBrother.logger.info("Running command: #{command.inspect}")
5
+ _system(command)
6
+ end
7
+
8
+ def _system(command)
9
+ current_fiber = Fiber.current
10
+
11
+ EventMachine.system(command) do |output, status|
12
+ current_fiber.resume(output, status.exitstatus)
13
+ end
14
+
15
+ return Fiber.yield
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,26 @@
1
+ module BigBrother
2
+ class StatusFile
3
+ def initialize(*filenames)
4
+ @path = File.join(BigBrother.config_dir, *filenames)
5
+ end
6
+
7
+ def content
8
+ File.read(@path).chomp
9
+ end
10
+
11
+ def create(reason)
12
+ FileUtils.mkdir_p(File.dirname(@path))
13
+ File.open(@path, 'w') do |file|
14
+ file.puts(reason)
15
+ end
16
+ end
17
+
18
+ def delete
19
+ FileUtils.rm(@path)
20
+ end
21
+
22
+ def exists?
23
+ File.exists?(@path)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,28 @@
1
+ module BigBrother
2
+ class Ticker
3
+
4
+ def self.pause(&block)
5
+ EM.cancel_timer(@timer)
6
+
7
+ while @outstanding_ticks > 0
8
+ EM::Synchrony.sleep(0.1)
9
+ end
10
+
11
+ block.call
12
+ schedule!
13
+ end
14
+
15
+ def self.schedule!
16
+ @outstanding_ticks = 0
17
+ @timer = EM::Synchrony.add_periodic_timer(0.1, &method(:tick))
18
+ end
19
+
20
+ def self.tick
21
+ @outstanding_ticks += 1
22
+ BigBrother.clusters.values.select(&:needs_check?).each do |cluster|
23
+ cluster.monitor_nodes
24
+ end
25
+ @outstanding_ticks -= 1
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,3 @@
1
+ module BigBrother
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,40 @@
1
+ # inlined from sinatra-synchrony
2
+ require 'sinatra/base'
3
+ require 'rack/fiber_pool'
4
+ require 'eventmachine'
5
+ require 'em-http-request'
6
+ require 'em-synchrony'
7
+ require 'em-resolv-replace'
8
+
9
+ module Sinatra
10
+ module Synchrony
11
+ def self.registered(app)
12
+ app.disable :threaded
13
+ end
14
+
15
+ def setup_sessions(builder)
16
+ builder.use Rack::FiberPool, {:rescue_exception => handle_exception } unless test?
17
+ super
18
+ end
19
+
20
+ def handle_exception
21
+ Proc.new do |env, e|
22
+ if settings.show_exceptions?
23
+ request = Sinatra::Request.new(env)
24
+ printer = Sinatra::ShowExceptions.new(proc{ raise e })
25
+ s, h, b = printer.call(env)
26
+ [s, h, b]
27
+ else
28
+ [500, {}, ""]
29
+ end
30
+ end
31
+ end
32
+
33
+ class << self
34
+ def overload_tcpsocket!
35
+ require 'sinatra/synchrony/tcpsocket'
36
+ end
37
+ end
38
+ end
39
+ register Synchrony
40
+ end
@@ -0,0 +1,20 @@
1
+ module Thin
2
+ module Backends
3
+ class TcpServerWithCallbacks < TcpServer
4
+ def initialize(host, port, options)
5
+ super(host, port)
6
+ end
7
+
8
+ def connect
9
+ super
10
+ Thin::Callbacks.after_connect_callbacks.each { |callback| callback.call }
11
+ end
12
+
13
+ def disconnect
14
+ Thin::Callbacks.before_disconnect_callbacks.each { |callback| callback.call }
15
+ super
16
+ end
17
+ end
18
+ end
19
+ end
20
+
@@ -0,0 +1,14 @@
1
+ module Thin
2
+ class CallbackRackHandler
3
+ def self.run(app, options)
4
+ server = ::Thin::Server.new(options[:Host] || '0.0.0.0',
5
+ options[:Port] || 8080,
6
+ app,
7
+ options)
8
+ yield server if block_given?
9
+ server.start
10
+ end
11
+ end
12
+ end
13
+
14
+ Rack::Handler.register 'thin-with-callbacks', Thin::CallbackRackHandler
@@ -0,0 +1,19 @@
1
+ module Thin
2
+ class Callbacks
3
+ def self.after_connect_callbacks
4
+ @after_connect_callbacks ||= []
5
+ end
6
+
7
+ def self.after_connect(&block)
8
+ after_connect_callbacks << block
9
+ end
10
+
11
+ def self.before_disconnect_callbacks
12
+ @before_disconnect_callbacks ||= []
13
+ end
14
+
15
+ def self.before_disconnect(&block)
16
+ before_disconnect_callbacks << block
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,127 @@
1
+ require 'spec_helper'
2
+
3
+ module BigBrother
4
+ describe App do
5
+ def app
6
+ App
7
+ end
8
+
9
+ describe "/" do
10
+ it "returns the list of configured clusters and their status" do
11
+ BigBrother.clusters['one'] = Factory.cluster(:name => 'one', :fwmark => 1)
12
+ BigBrother.clusters['two'] = Factory.cluster(:name => 'two', :fwmark => 2)
13
+ BigBrother.clusters['three'] = Factory.cluster(:name => 'three', :fwmark => 3)
14
+ BigBrother.clusters['three'].start_monitoring!
15
+ BigBrother.clusters['four'] = Factory.cluster(:name => 'four', :fwmark => 4)
16
+
17
+ get "/"
18
+ last_response.status.should == 200
19
+ last_response.body.should include("one (1): not running")
20
+ last_response.body.should include("two (2): not running")
21
+ last_response.body.should include("three (3): running")
22
+ last_response.body.should include("four (4): not running")
23
+ end
24
+ end
25
+
26
+ describe "GET /cluster/:name" do
27
+ it "returns 'Running: false' when the cluster isn't running" do
28
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
29
+
30
+ get "/cluster/test"
31
+
32
+ last_response.status.should == 200
33
+ last_response.body.should == "Running: false"
34
+ end
35
+
36
+ it "returns 'Running: true' when the cluster is running" do
37
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
38
+
39
+ put "/cluster/test"
40
+ get "/cluster/test"
41
+
42
+ last_response.status.should == 200
43
+ last_response.body.should == "Running: true"
44
+ end
45
+
46
+ it "returns a 404 http status when the cluster is not found" do
47
+ get "/cluster/not_found"
48
+
49
+ last_response.status.should == 404
50
+ end
51
+ end
52
+
53
+ describe "PUT /cluster/:name" do
54
+ it "marks the cluster as monitored" do
55
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
56
+
57
+ put "/cluster/test"
58
+
59
+ last_response.status.should == 200
60
+ last_response.body.should == ""
61
+ BigBrother.clusters['test'].should be_monitored
62
+ end
63
+
64
+ it "only starts monitoring the cluster once" do
65
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
66
+
67
+ put "/cluster/test"
68
+ last_response.status.should == 200
69
+
70
+ put "/cluster/test"
71
+ last_response.status.should == 304
72
+ end
73
+
74
+ it "returns 'not found' if the cluster does not exist" do
75
+ put "/cluster/test"
76
+
77
+ last_response.status.should == 404
78
+ last_response.body.should == "Cluster test not found"
79
+ end
80
+
81
+ it "populates IPVS" do
82
+ first = Factory.node(:address => '127.0.0.1')
83
+ second = Factory.node(:address => '127.0.0.2')
84
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test', :fwmark => 100, :scheduler => 'wrr', :nodes => [first, second])
85
+
86
+ put "/cluster/test"
87
+
88
+ last_response.status.should == 200
89
+ last_response.body.should == ""
90
+ BigBrother.clusters['test'].should be_monitored
91
+ @recording_executor.commands.first.should == "ipvsadm --add-service --fwmark-service 100 --scheduler wrr"
92
+ @recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 100")
93
+ @recording_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 100")
94
+ end
95
+ end
96
+
97
+ describe "DELETE /cluster/:name" do
98
+ it "marks the cluster as no longer monitored" do
99
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
100
+ BigBrother.clusters['test'].start_monitoring!
101
+
102
+ delete "/cluster/test"
103
+
104
+ last_response.status.should == 200
105
+ last_response.body.should == ""
106
+ BigBrother.clusters['test'].should_not be_monitored
107
+ end
108
+
109
+ it "only stops monitoring the cluster once" do
110
+ BigBrother.clusters['test'] = Factory.cluster(:name => 'test')
111
+
112
+ delete "/cluster/test"
113
+
114
+ last_response.status.should == 304
115
+ last_response.body.should == ""
116
+ BigBrother.clusters['test'].should_not be_monitored
117
+ end
118
+
119
+ it "returns 'not found' if the cluster does not exist" do
120
+ delete "/cluster/test"
121
+
122
+ last_response.status.should == 404
123
+ last_response.body.should == "Cluster test not found"
124
+ end
125
+ end
126
+ end
127
+ end