big_brother 0.6.8 → 0.8.7
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -0
- data/Changelog.md +5 -0
- data/big_brother.gemspec +1 -1
- data/lib/big_brother.rb +4 -0
- data/lib/big_brother/active_active_cluster.rb +157 -0
- data/lib/big_brother/active_passive_cluster.rb +61 -0
- data/lib/big_brother/cluster.rb +28 -13
- data/lib/big_brother/cluster_collection.rb +3 -0
- data/lib/big_brother/cluster_factory.rb +16 -0
- data/lib/big_brother/configuration.rb +21 -3
- data/lib/big_brother/health_fetcher.rb +12 -0
- data/lib/big_brother/node.rb +35 -10
- data/lib/big_brother/version.rb +1 -1
- data/lib/resources/config_schema.yml +92 -0
- data/spec/big_brother/active_active_cluster_spec.rb +437 -0
- data/spec/big_brother/active_passive_cluster_spec.rb +172 -0
- data/spec/big_brother/app_spec.rb +13 -11
- data/spec/big_brother/cluster_collection_spec.rb +26 -0
- data/spec/big_brother/cluster_factory_spec.rb +23 -0
- data/spec/big_brother/cluster_spec.rb +60 -18
- data/spec/big_brother/configuration_spec.rb +72 -27
- data/spec/big_brother/health_fetcher_spec.rb +47 -2
- data/spec/big_brother/node_spec.rb +42 -68
- data/spec/big_brother/ticker_spec.rb +6 -2
- data/spec/big_brother_spec.rb +85 -55
- data/spec/support/example_config.yml +65 -39
- data/spec/support/factories/cluster_factory.rb +9 -1
- data/spec/support/null_logger.rb +9 -0
- metadata +30 -25
data/.travis.yml
CHANGED
data/Changelog.md
ADDED
@@ -0,0 +1,5 @@
|
|
1
|
+
0.6.8:
|
2
|
+
|
3
|
+
- Enable active/passive mode, an optional `backend_mode: "active_passive"` configuration directive is added to the big brother cluster config.
|
4
|
+
- Active/Passive nodes now take a `priority` field that is used to select the active backend for the cluster.
|
5
|
+
- The active backend for an active/passive cluster is the backend with the lowest priority and positive health response. Should two backends have the same priority, the lowest backend IP address is selected. If none of the backends is healthy, the last active node's health is updated to weight of 0 in ipvs.
|
data/big_brother.gemspec
CHANGED
@@ -24,10 +24,10 @@ Gem::Specification.new do |gem|
|
|
24
24
|
gem.add_dependency "em-synchrony", "~> 1.0"
|
25
25
|
gem.add_dependency "em-resolv-replace", "~> 1.1"
|
26
26
|
gem.add_dependency "em-syslog", "~> 0.0.2"
|
27
|
+
gem.add_dependency "kwalify", "~> 0.7.2"
|
27
28
|
|
28
29
|
gem.add_development_dependency "rspec", "~> 2.9.0"
|
29
30
|
gem.add_development_dependency "rack-test", "~> 0.6.1"
|
30
31
|
gem.add_development_dependency "rake"
|
31
32
|
gem.add_development_dependency "rake_commit", "~> 0.13"
|
32
|
-
gem.add_development_dependency "vagrant"
|
33
33
|
end
|
data/lib/big_brother.rb
CHANGED
@@ -4,11 +4,15 @@ require 'em-synchrony/em-http'
|
|
4
4
|
require 'em/syslog'
|
5
5
|
require 'thin'
|
6
6
|
require 'yaml'
|
7
|
+
require 'kwalify'
|
7
8
|
|
8
9
|
require 'sinatra/synchrony'
|
9
10
|
|
10
11
|
require 'big_brother/app'
|
11
12
|
require 'big_brother/cluster'
|
13
|
+
require 'big_brother/active_passive_cluster'
|
14
|
+
require 'big_brother/active_active_cluster'
|
15
|
+
require 'big_brother/cluster_factory'
|
12
16
|
require 'big_brother/cluster_collection'
|
13
17
|
require 'big_brother/configuration'
|
14
18
|
require 'big_brother/health_fetcher'
|
@@ -0,0 +1,157 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class ActiveActiveCluster < BigBrother::Cluster
|
3
|
+
attr_reader :interpol_node, :max_down_ticks, :offset, :remote_nodes, :local_nodes, :non_egress_locations
|
4
|
+
|
5
|
+
def initialize(name, attributes={})
|
6
|
+
super(name, attributes)
|
7
|
+
interpol_nodes, local_nodes = @nodes.partition { |node| node.interpol? }
|
8
|
+
@nodes = @local_nodes = local_nodes
|
9
|
+
@interpol_node = interpol_nodes.first
|
10
|
+
@remote_nodes = []
|
11
|
+
@max_down_ticks = attributes.fetch(:max_down_ticks, 0)
|
12
|
+
@offset = attributes.fetch(:offset, 10_000)
|
13
|
+
@non_egress_locations = *attributes.fetch(:non_egress_locations, [])
|
14
|
+
end
|
15
|
+
|
16
|
+
def start_monitoring!
|
17
|
+
BigBrother.logger.info "Starting monitoring on active/active cluster #{to_s}"
|
18
|
+
BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
|
19
|
+
BigBrother.ipvs.start_cluster(_relay_fwmark, @scheduler)
|
20
|
+
local_nodes.each do |node|
|
21
|
+
BigBrother.ipvs.start_node(@fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
|
22
|
+
BigBrother.ipvs.start_node(_relay_fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
|
23
|
+
end
|
24
|
+
|
25
|
+
@monitored = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def stop_monitoring!
|
29
|
+
super
|
30
|
+
BigBrother.ipvs.stop_cluster(_relay_fwmark)
|
31
|
+
end
|
32
|
+
|
33
|
+
def monitor_nodes
|
34
|
+
super
|
35
|
+
|
36
|
+
fresh_remote_nodes = _fetch_remote_nodes
|
37
|
+
remote_nodes.each do |node|
|
38
|
+
if new_node = fresh_remote_nodes[node.address]
|
39
|
+
next if new_node.weight == node.weight
|
40
|
+
BigBrother.ipvs.edit_node(fwmark, node.address, new_node.weight)
|
41
|
+
node.weight = new_node.weight
|
42
|
+
else
|
43
|
+
_adjust_or_remove_remote_node(node)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
_add_remote_nodes(fresh_remote_nodes.values - remote_nodes)
|
48
|
+
end
|
49
|
+
|
50
|
+
def synchronize!
|
51
|
+
ipvs_state = BigBrother.ipvs.running_configuration
|
52
|
+
@remote_nodes = _fetch_remote_nodes.values if @remote_nodes == []
|
53
|
+
if ipvs_state.has_key?(fwmark.to_s)
|
54
|
+
resume_monitoring!
|
55
|
+
|
56
|
+
running_nodes = ipvs_state[fwmark.to_s]
|
57
|
+
_remove_nodes(running_nodes - cluster_nodes)
|
58
|
+
_add_nodes(cluster_nodes - running_nodes, fwmark)
|
59
|
+
_add_nodes(local_cluster_nodes - running_nodes, _relay_fwmark)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def cluster_nodes
|
64
|
+
(nodes + remote_nodes).map(&:address)
|
65
|
+
end
|
66
|
+
|
67
|
+
def local_cluster_nodes
|
68
|
+
nodes.map(&:address)
|
69
|
+
end
|
70
|
+
|
71
|
+
def incorporate_state(cluster)
|
72
|
+
ipvs_state = BigBrother.ipvs.running_configuration
|
73
|
+
if ipvs_state[fwmark.to_s] && ipvs_state[_relay_fwmark.to_s].nil?
|
74
|
+
BigBrother.logger.info "Adding new active/active LB node #{to_s}"
|
75
|
+
BigBrother.ipvs.start_cluster(_relay_fwmark, @scheduler)
|
76
|
+
end
|
77
|
+
|
78
|
+
if ipvs_state[fwmark.to_s] && ipvs_state.fetch(_relay_fwmark.to_s, []).empty?
|
79
|
+
nodes.each do |node|
|
80
|
+
actual_node = cluster.find_node(node.address, node.port)
|
81
|
+
BigBrother.ipvs.start_node(_relay_fwmark, actual_node.address, actual_node.weight)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
BigBrother.logger.info "Merging in new active/active cluster #{to_s}"
|
86
|
+
|
87
|
+
@remote_nodes = cluster.remote_nodes if cluster.is_a?(BigBrother::ActiveActiveCluster)
|
88
|
+
|
89
|
+
super(cluster)
|
90
|
+
end
|
91
|
+
|
92
|
+
def stop_relay_fwmark
|
93
|
+
nodes.each do |node|
|
94
|
+
BigBrother.ipvs.stop_node(_relay_fwmark, node.address)
|
95
|
+
end
|
96
|
+
|
97
|
+
BigBrother.ipvs.stop_cluster(_relay_fwmark)
|
98
|
+
end
|
99
|
+
|
100
|
+
def _relay_fwmark
|
101
|
+
fwmark + offset
|
102
|
+
end
|
103
|
+
|
104
|
+
def _add_nodes(addresses, fwmark)
|
105
|
+
addresses.each do |address|
|
106
|
+
BigBrother.logger.info "Adding #{address} to active/active cluster #{self}"
|
107
|
+
BigBrother.ipvs.start_node(fwmark, address, 0)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def _adjust_or_remove_remote_node(node)
|
112
|
+
if node.down_tick_count >= max_down_ticks
|
113
|
+
BigBrother.ipvs.stop_node(fwmark, node.address)
|
114
|
+
remote_nodes.delete(node)
|
115
|
+
else
|
116
|
+
BigBrother.ipvs.edit_node(fwmark, node.address, 0)
|
117
|
+
node.weight = 0
|
118
|
+
node.down_tick_count += 1
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def _fetch_remote_nodes
|
123
|
+
return {} if interpol_node.nil?
|
124
|
+
|
125
|
+
regular_remote_cluster = BigBrother::HealthFetcher.interpol_status(interpol_node, fwmark)
|
126
|
+
relay_remote_cluster = BigBrother::HealthFetcher.interpol_status(interpol_node, _relay_fwmark)
|
127
|
+
|
128
|
+
return {} if regular_remote_cluster.empty? || relay_remote_cluster.empty?
|
129
|
+
|
130
|
+
regular_remote_cluster.each_with_object({}) do |node, hsh|
|
131
|
+
next if self.non_egress_locations.include?(node['lb_source_location'])
|
132
|
+
|
133
|
+
hsh[node['lb_ip_address']] = BigBrother::Node.new(:address => node['lb_ip_address'], :weight => node['health'])
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def _add_remote_nodes(nodes)
|
138
|
+
nodes.each do |node|
|
139
|
+
BigBrother.ipvs.start_node(fwmark, node.address, node.weight)
|
140
|
+
@remote_nodes << node
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def _remove_nodes(addresses)
|
145
|
+
addresses.each do |address|
|
146
|
+
BigBrother.logger.info "Removing #{address} from active/active cluster #{self}"
|
147
|
+
BigBrother.ipvs.stop_node(fwmark, address)
|
148
|
+
BigBrother.ipvs.stop_node(_relay_fwmark, address)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def _update_node(node, new_weight)
|
153
|
+
BigBrother.ipvs.edit_node(fwmark, node.address, new_weight)
|
154
|
+
BigBrother.ipvs.edit_node(_relay_fwmark, node.address, new_weight)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class ActivePassiveCluster < Cluster
|
3
|
+
|
4
|
+
def start_monitoring!
|
5
|
+
BigBrother.logger.info "starting monitoring on cluster #{to_s}"
|
6
|
+
BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
|
7
|
+
BigBrother.ipvs.start_node(@fwmark, active_node.address, BigBrother::Node::INITIAL_WEIGHT)
|
8
|
+
|
9
|
+
@monitored = true
|
10
|
+
end
|
11
|
+
|
12
|
+
def active_node
|
13
|
+
@current_active_node ||= @nodes.sort.first
|
14
|
+
end
|
15
|
+
|
16
|
+
def synchronize!
|
17
|
+
ipvs_state = BigBrother.ipvs.running_configuration
|
18
|
+
if ipvs_state.has_key?(fwmark.to_s)
|
19
|
+
resume_monitoring!
|
20
|
+
|
21
|
+
running_active_node_address = ipvs_state[fwmark.to_s].first
|
22
|
+
if running_active_node_address != active_node.address
|
23
|
+
BigBrother.ipvs.stop_node(fwmark, running_active_node_address)
|
24
|
+
BigBrother.ipvs.start_node(fwmark, active_node.address, active_node.weight)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def monitor_nodes
|
30
|
+
@last_check = Time.now
|
31
|
+
@current_active_node = active_node
|
32
|
+
proposed_active_node = @nodes.reject do |node|
|
33
|
+
weight = node.monitor(self).to_i
|
34
|
+
_modify_current_active_node_weight(node, weight)
|
35
|
+
node.weight = weight
|
36
|
+
node.weight.zero?
|
37
|
+
end.sort.first
|
38
|
+
|
39
|
+
if proposed_active_node != @current_active_node && !proposed_active_node.nil?
|
40
|
+
_modify_active_node(@current_active_node, proposed_active_node)
|
41
|
+
@current_active_node = proposed_active_node
|
42
|
+
end
|
43
|
+
|
44
|
+
_check_downpage if has_downpage?
|
45
|
+
_notify_nagios if nagios
|
46
|
+
end
|
47
|
+
|
48
|
+
def _modify_current_active_node_weight(node, weight)
|
49
|
+
return unless node == @current_active_node
|
50
|
+
if @current_active_node.weight != weight
|
51
|
+
BigBrother.ipvs.edit_node(fwmark, @current_active_node.address, weight)
|
52
|
+
@current_active_node.weight = weight
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def _modify_active_node(current_active_node, proposed_active_node)
|
57
|
+
BigBrother.ipvs.stop_node(fwmark, current_active_node.address)
|
58
|
+
BigBrother.ipvs.start_node(fwmark, proposed_active_node.address, proposed_active_node.weight)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/lib/big_brother/cluster.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module BigBrother
|
2
2
|
class Cluster
|
3
|
-
attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name, :ramp_up_time, :nagios
|
3
|
+
attr_reader :fwmark, :scheduler, :check_interval, :nodes, :name, :ramp_up_time, :nagios, :backend_mode
|
4
4
|
|
5
5
|
def initialize(name, attributes = {})
|
6
6
|
@name = name
|
@@ -15,6 +15,7 @@ module BigBrother
|
|
15
15
|
@ramp_up_time = attributes.fetch(:ramp_up_time, 60)
|
16
16
|
@has_downpage = attributes[:has_downpage]
|
17
17
|
@nagios = attributes[:nagios]
|
18
|
+
@backend_mode = attributes[:backend_mode]
|
18
19
|
end
|
19
20
|
|
20
21
|
def _coerce_node(node_config)
|
@@ -42,17 +43,17 @@ module BigBrother
|
|
42
43
|
end
|
43
44
|
|
44
45
|
def start_monitoring!
|
45
|
-
BigBrother.logger.info "
|
46
|
+
BigBrother.logger.info "Starting monitoring on cluster #{to_s}"
|
46
47
|
BigBrother.ipvs.start_cluster(@fwmark, @scheduler)
|
47
48
|
@nodes.each do |node|
|
48
|
-
BigBrother.ipvs.start_node(@fwmark, node.address,
|
49
|
+
BigBrother.ipvs.start_node(@fwmark, node.address, BigBrother::Node::INITIAL_WEIGHT)
|
49
50
|
end
|
50
51
|
|
51
52
|
@monitored = true
|
52
53
|
end
|
53
54
|
|
54
55
|
def stop_monitoring!
|
55
|
-
BigBrother.logger.info "
|
56
|
+
BigBrother.logger.info "Stopping monitoring on cluster #{to_s}"
|
56
57
|
BigBrother.ipvs.stop_cluster(@fwmark)
|
57
58
|
|
58
59
|
@monitored = false
|
@@ -60,7 +61,7 @@ module BigBrother
|
|
60
61
|
end
|
61
62
|
|
62
63
|
def resume_monitoring!
|
63
|
-
BigBrother.logger.info "
|
64
|
+
BigBrother.logger.info "Resuming monitoring on cluster #{to_s}"
|
64
65
|
@monitored = true
|
65
66
|
end
|
66
67
|
|
@@ -70,13 +71,15 @@ module BigBrother
|
|
70
71
|
resume_monitoring!
|
71
72
|
|
72
73
|
running_nodes = ipvs_state[fwmark.to_s]
|
73
|
-
cluster_nodes = nodes.map(&:address)
|
74
|
-
|
75
74
|
_remove_nodes(running_nodes - cluster_nodes)
|
76
75
|
_add_nodes(cluster_nodes - running_nodes)
|
77
76
|
end
|
78
77
|
end
|
79
78
|
|
79
|
+
def cluster_nodes
|
80
|
+
nodes.map(&:address)
|
81
|
+
end
|
82
|
+
|
80
83
|
def needs_check?
|
81
84
|
return false unless monitored?
|
82
85
|
@last_check + @check_interval < Time.now
|
@@ -84,7 +87,14 @@ module BigBrother
|
|
84
87
|
|
85
88
|
def monitor_nodes
|
86
89
|
@last_check = Time.now
|
87
|
-
|
90
|
+
return unless monitored?
|
91
|
+
@nodes.each do |node|
|
92
|
+
new_weight = node.monitor(self)
|
93
|
+
if new_weight != node.weight
|
94
|
+
_update_node(node, new_weight)
|
95
|
+
node.weight = new_weight
|
96
|
+
end
|
97
|
+
end
|
88
98
|
|
89
99
|
_check_downpage if has_downpage?
|
90
100
|
_notify_nagios if nagios
|
@@ -110,19 +120,20 @@ module BigBrother
|
|
110
120
|
nodes.each do |node|
|
111
121
|
node.incorporate_state(another_cluster.find_node(node.address, node.port))
|
112
122
|
end
|
123
|
+
|
113
124
|
self
|
114
125
|
end
|
115
126
|
|
116
127
|
def _add_nodes(addresses)
|
117
128
|
addresses.each do |address|
|
118
|
-
BigBrother.logger.info "
|
119
|
-
BigBrother.ipvs.start_node(fwmark, address,
|
129
|
+
BigBrother.logger.info "Adding #{address} to cluster #{self}"
|
130
|
+
BigBrother.ipvs.start_node(fwmark, address, 0)
|
120
131
|
end
|
121
132
|
end
|
122
133
|
|
123
134
|
def _add_maintenance_node
|
124
|
-
BigBrother.logger.info "
|
125
|
-
BigBrother.ipvs.start_node(fwmark, '
|
135
|
+
BigBrother.logger.info "Adding downpage to cluster #{self}"
|
136
|
+
BigBrother.ipvs.start_node(fwmark, '169.254.254.254', 1)
|
126
137
|
end
|
127
138
|
|
128
139
|
def _check_downpage
|
@@ -155,9 +166,13 @@ module BigBrother
|
|
155
166
|
|
156
167
|
def _remove_nodes(addresses)
|
157
168
|
addresses.each do |address|
|
158
|
-
BigBrother.logger.info "
|
169
|
+
BigBrother.logger.info "Removing #{address} to cluster #{self}"
|
159
170
|
BigBrother.ipvs.stop_node(fwmark, address)
|
160
171
|
end
|
161
172
|
end
|
173
|
+
|
174
|
+
def _update_node(node, new_weight)
|
175
|
+
BigBrother.ipvs.edit_node(fwmark, node.address, new_weight)
|
176
|
+
end
|
162
177
|
end
|
163
178
|
end
|
@@ -15,6 +15,9 @@ module BigBrother
|
|
15
15
|
end
|
16
16
|
new_clusters.each do |cluster_name, cluster|
|
17
17
|
if @clusters.key?(cluster_name)
|
18
|
+
current_cluster = @clusters[cluster_name]
|
19
|
+
current_cluster.stop_relay_fwmark if current_cluster.is_a?(BigBrother::ActiveActiveCluster) && !cluster.is_a?(BigBrother::ActiveActiveCluster)
|
20
|
+
|
18
21
|
@clusters[cluster_name] = cluster.incorporate_state(@clusters[cluster_name])
|
19
22
|
else
|
20
23
|
@clusters[cluster_name] = cluster
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module BigBrother
|
2
|
+
class ClusterFactory
|
3
|
+
|
4
|
+
ACTIVE_PASSIVE_CLUSTER = 'active_passive'
|
5
|
+
ACTIVE_ACTIVE_CLUSTER = 'active_active'
|
6
|
+
|
7
|
+
CLUSTERS = {
|
8
|
+
ACTIVE_PASSIVE_CLUSTER => ActivePassiveCluster,
|
9
|
+
ACTIVE_ACTIVE_CLUSTER => ActiveActiveCluster,
|
10
|
+
}
|
11
|
+
|
12
|
+
def self.create_cluster(name, attributes)
|
13
|
+
CLUSTERS.fetch(attributes[:backend_mode], BigBrother::Cluster).new(name, attributes)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -4,11 +4,15 @@ module BigBrother
|
|
4
4
|
|
5
5
|
def self.from_file(config_file)
|
6
6
|
config = YAML.load_file(config_file)
|
7
|
+
return {} unless _valid?(config)
|
8
|
+
|
9
|
+
configured_clusters = config.fetch("clusters")
|
7
10
|
defaults = config.delete(GLOBAL_CONFIG_KEY)
|
8
11
|
|
9
|
-
|
10
|
-
cluster_details = _apply_defaults(defaults,
|
11
|
-
|
12
|
+
configured_clusters.inject({}) do |clusters, cluster|
|
13
|
+
cluster_details = _apply_defaults(defaults, cluster)
|
14
|
+
cluster_name = cluster.fetch("cluster_name")
|
15
|
+
clusters.merge(cluster_name => BigBrother::ClusterFactory.create_cluster(cluster_name, _deeply_symbolize_keys(cluster_details)))
|
12
16
|
end
|
13
17
|
end
|
14
18
|
|
@@ -31,5 +35,19 @@ module BigBrother
|
|
31
35
|
oldval.is_a?(Hash) && newval.is_a?(Hash) ? _apply_defaults(oldval, newval) : newval
|
32
36
|
end
|
33
37
|
end
|
38
|
+
|
39
|
+
def self._valid?(config)
|
40
|
+
schema_path = File.join(File.dirname(__FILE__), "../resources", "config_schema.yml")
|
41
|
+
schema = YAML.load_file(schema_path)
|
42
|
+
validator = Kwalify::Validator.new(schema)
|
43
|
+
errors = validator.validate(config)
|
44
|
+
valid = !(errors && !errors.empty?)
|
45
|
+
|
46
|
+
unless valid
|
47
|
+
errors.each { |err| BigBrother.logger.info("- [#{err.path}] #{err.message}") }
|
48
|
+
end
|
49
|
+
|
50
|
+
valid
|
51
|
+
end
|
34
52
|
end
|
35
53
|
end
|