big_brother 0.6.8 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +1 -0
- data/Changelog.md +5 -0
- data/big_brother.gemspec +1 -1
- data/lib/big_brother.rb +4 -0
- data/lib/big_brother/active_active_cluster.rb +157 -0
- data/lib/big_brother/active_passive_cluster.rb +61 -0
- data/lib/big_brother/cluster.rb +28 -13
- data/lib/big_brother/cluster_collection.rb +3 -0
- data/lib/big_brother/cluster_factory.rb +16 -0
- data/lib/big_brother/configuration.rb +21 -3
- data/lib/big_brother/health_fetcher.rb +12 -0
- data/lib/big_brother/node.rb +35 -10
- data/lib/big_brother/version.rb +1 -1
- data/lib/resources/config_schema.yml +92 -0
- data/spec/big_brother/active_active_cluster_spec.rb +437 -0
- data/spec/big_brother/active_passive_cluster_spec.rb +172 -0
- data/spec/big_brother/app_spec.rb +13 -11
- data/spec/big_brother/cluster_collection_spec.rb +26 -0
- data/spec/big_brother/cluster_factory_spec.rb +23 -0
- data/spec/big_brother/cluster_spec.rb +60 -18
- data/spec/big_brother/configuration_spec.rb +72 -27
- data/spec/big_brother/health_fetcher_spec.rb +47 -2
- data/spec/big_brother/node_spec.rb +42 -68
- data/spec/big_brother/ticker_spec.rb +6 -2
- data/spec/big_brother_spec.rb +85 -55
- data/spec/support/example_config.yml +65 -39
- data/spec/support/factories/cluster_factory.rb +9 -1
- data/spec/support/null_logger.rb +9 -0
- metadata +30 -25
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
1
3
|
module BigBrother
|
2
4
|
class HealthFetcher
|
3
5
|
def self.current_health(address, port, path)
|
@@ -8,6 +10,16 @@ module BigBrother
|
|
8
10
|
response.response_header.status == 200 ? _parse_health(response) : 0
|
9
11
|
end
|
10
12
|
|
13
|
+
def self.interpol_status(interpol_node, fwmark)
|
14
|
+
url = "http://#{interpol_node.address}:#{interpol_node.port}/fwmark/#{fwmark}"
|
15
|
+
|
16
|
+
BigBrother.logger.debug("Fetching health from #{url}")
|
17
|
+
response = EventMachine::HttpRequest.new(url).get
|
18
|
+
response.response_header.status == 200 ? JSON.parse(response.response) : []
|
19
|
+
rescue JSON::ParserError
|
20
|
+
[]
|
21
|
+
end
|
22
|
+
|
11
23
|
def self._parse_health(response)
|
12
24
|
if response.response_header.has_key?('X_HEALTH')
|
13
25
|
response.response_header['X_HEALTH'].to_i
|
data/lib/big_brother/node.rb
CHANGED
@@ -2,7 +2,9 @@ require 'net/http'
|
|
2
2
|
|
3
3
|
module BigBrother
|
4
4
|
class Node
|
5
|
-
attr_reader :address, :port, :path, :start_time, :
|
5
|
+
attr_reader :address, :port, :path, :start_time, :priority, :max_weight
|
6
|
+
attr_accessor :weight, :down_tick_count
|
7
|
+
INITIAL_WEIGHT = 1
|
6
8
|
|
7
9
|
def initialize(attributes={})
|
8
10
|
@address = attributes[:address]
|
@@ -10,6 +12,10 @@ module BigBrother
|
|
10
12
|
@path = attributes[:path]
|
11
13
|
@weight = attributes[:weight]
|
12
14
|
@start_time = attributes.fetch(:start_time, Time.now.to_i)
|
15
|
+
@priority = attributes.fetch(:priority, 0)
|
16
|
+
@interpol = attributes.fetch(:interpol, false)
|
17
|
+
@max_weight = attributes[:max_weight]
|
18
|
+
@down_tick_count = 0
|
13
19
|
end
|
14
20
|
|
15
21
|
def age
|
@@ -27,26 +33,45 @@ module BigBrother
|
|
27
33
|
@weight = nil
|
28
34
|
end
|
29
35
|
|
30
|
-
def
|
31
|
-
|
32
|
-
return unless cluster.monitored?
|
33
|
-
if new_weight != @weight
|
34
|
-
BigBrother.ipvs.edit_node(cluster.fwmark, address, new_weight)
|
35
|
-
@weight = new_weight
|
36
|
-
end
|
36
|
+
def interpol?
|
37
|
+
@interpol
|
37
38
|
end
|
38
39
|
|
39
40
|
def ==(other)
|
40
41
|
address == other.address && port == other.port
|
41
42
|
end
|
43
|
+
alias eql? ==
|
44
|
+
|
45
|
+
def hash
|
46
|
+
[@address, @port].hash
|
47
|
+
end
|
48
|
+
|
49
|
+
def <=>(other)
|
50
|
+
return 1 if self.weight.to_i.zero?
|
51
|
+
return -1 if other.weight.to_i.zero?
|
52
|
+
comparison = self.priority <=> other.priority
|
53
|
+
if comparison.zero?
|
54
|
+
self.address <=> other.address
|
55
|
+
else
|
56
|
+
comparison
|
57
|
+
end
|
58
|
+
end
|
42
59
|
|
43
|
-
def
|
60
|
+
def monitor(cluster)
|
44
61
|
if cluster.up_file_exists?
|
45
62
|
100
|
46
63
|
elsif cluster.down_file_exists?
|
47
64
|
0
|
48
65
|
else
|
49
|
-
_weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time)
|
66
|
+
_cap_weight(_weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time))
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def _cap_weight(health)
|
71
|
+
if !@max_weight.nil? && @max_weight.is_a?(Integer) && @max_weight > 0 && @max_weight < health
|
72
|
+
@max_weight
|
73
|
+
else
|
74
|
+
health
|
50
75
|
end
|
51
76
|
end
|
52
77
|
|
data/lib/big_brother/version.rb
CHANGED
@@ -0,0 +1,92 @@
|
|
1
|
+
type: map
|
2
|
+
mapping:
|
3
|
+
"_big_brother":
|
4
|
+
type: map
|
5
|
+
mapping:
|
6
|
+
"offset":
|
7
|
+
type: int
|
8
|
+
"check_interval":
|
9
|
+
type: int
|
10
|
+
"max_down_ticks":
|
11
|
+
type: int
|
12
|
+
"backend_mode":
|
13
|
+
type: str
|
14
|
+
enum: [active_passive, active_active]
|
15
|
+
"scheduler":
|
16
|
+
type: str
|
17
|
+
enum: [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
|
18
|
+
"ramp_up_time":
|
19
|
+
type: int
|
20
|
+
"has_downpage":
|
21
|
+
type: bool
|
22
|
+
"nagios":
|
23
|
+
type: map
|
24
|
+
mapping:
|
25
|
+
"server":
|
26
|
+
type: str
|
27
|
+
"check":
|
28
|
+
type: str
|
29
|
+
"host":
|
30
|
+
type: str
|
31
|
+
"clusters":
|
32
|
+
type: seq
|
33
|
+
required: yes
|
34
|
+
sequence:
|
35
|
+
- type: map
|
36
|
+
mapping:
|
37
|
+
"non_egress_locations":
|
38
|
+
type: seq
|
39
|
+
sequence:
|
40
|
+
- type: str
|
41
|
+
"cluster_name":
|
42
|
+
type: str
|
43
|
+
required: yes
|
44
|
+
"check_interval":
|
45
|
+
type: int
|
46
|
+
"max_down_ticks":
|
47
|
+
type: int
|
48
|
+
"backend_mode":
|
49
|
+
type: str
|
50
|
+
enum: [active_passive, active_active]
|
51
|
+
"scheduler":
|
52
|
+
type: str
|
53
|
+
enum: [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
|
54
|
+
required: yes
|
55
|
+
"fwmark":
|
56
|
+
type: int
|
57
|
+
required: yes
|
58
|
+
"ramp_up_time":
|
59
|
+
type: int
|
60
|
+
"has_downpage":
|
61
|
+
type: bool
|
62
|
+
"nagios":
|
63
|
+
type: map
|
64
|
+
mapping:
|
65
|
+
"server":
|
66
|
+
type: str
|
67
|
+
"check":
|
68
|
+
type: str
|
69
|
+
"host":
|
70
|
+
type: str
|
71
|
+
"nodes":
|
72
|
+
type: seq
|
73
|
+
required: yes
|
74
|
+
sequence:
|
75
|
+
- type: map
|
76
|
+
mapping:
|
77
|
+
"address":
|
78
|
+
type: str
|
79
|
+
pattern: /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
|
80
|
+
required: yes
|
81
|
+
"interpol":
|
82
|
+
type: bool
|
83
|
+
"max_weight":
|
84
|
+
type: int
|
85
|
+
"priority":
|
86
|
+
type: int
|
87
|
+
"port":
|
88
|
+
type: int
|
89
|
+
range: { max: 65535, min: 1 }
|
90
|
+
"path":
|
91
|
+
type: str
|
92
|
+
pattern: /^\//
|
@@ -0,0 +1,437 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe BigBrother::ActiveActiveCluster do
|
4
|
+
before { BigBrother::HealthFetcher.stub(:current_health).and_return(10) }
|
5
|
+
|
6
|
+
describe '#start_monitoring!' do
|
7
|
+
it 'starts all non interpol nodes' do
|
8
|
+
cluster = Factory.active_active_cluster(
|
9
|
+
:fwmark => 100,
|
10
|
+
:scheduler => 'wrr',
|
11
|
+
:nodes => [
|
12
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
13
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
14
|
+
Factory.node(:interpol => true, :address => '127.0.0.3'),
|
15
|
+
],
|
16
|
+
)
|
17
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
18
|
+
cluster.start_monitoring!
|
19
|
+
|
20
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1')
|
21
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 1')
|
22
|
+
@stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.3 --ipip --weight 1')
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'starts all relay fwmark service' do
|
26
|
+
cluster = Factory.active_active_cluster(
|
27
|
+
:fwmark => 100,
|
28
|
+
:scheduler => 'wrr',
|
29
|
+
:offset => 10000,
|
30
|
+
:nodes => [
|
31
|
+
Factory.node(:interpol => true, :address => '127.0.0.3'),
|
32
|
+
],
|
33
|
+
)
|
34
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
35
|
+
cluster.start_monitoring!
|
36
|
+
|
37
|
+
@stub_executor.commands.should include('ipvsadm --add-service --fwmark-service 10100 --scheduler wrr')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'starts all local_nodes on relay fwmark' do
|
41
|
+
cluster = Factory.active_active_cluster(
|
42
|
+
:fwmark => 100,
|
43
|
+
:scheduler => 'wrr',
|
44
|
+
:offset => 10000,
|
45
|
+
:nodes => [
|
46
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
47
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
48
|
+
Factory.node(:interpol => true, :address => '127.0.0.3'),
|
49
|
+
],
|
50
|
+
)
|
51
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
52
|
+
cluster.start_monitoring!
|
53
|
+
|
54
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 1')
|
55
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.2 --ipip --weight 1')
|
56
|
+
@stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.3 --ipip --weight 1')
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'does not attempt to retrieve interpol status if there is no interpol node' do
|
60
|
+
cluster = Factory.active_active_cluster(
|
61
|
+
:fwmark => 100,
|
62
|
+
:scheduler => 'wrr',
|
63
|
+
:nodes => [
|
64
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
65
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
66
|
+
],
|
67
|
+
)
|
68
|
+
|
69
|
+
BigBrother::HealthFetcher.should_not_receive(:interpol_status)
|
70
|
+
|
71
|
+
cluster.start_monitoring!
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'does not start interpol nodes when the remote relay cluster is non-existent' do
|
75
|
+
node = Factory.node(:interpol => true, :address => '127.0.0.3')
|
76
|
+
cluster = Factory.active_active_cluster(
|
77
|
+
:fwmark => 100,
|
78
|
+
:scheduler => 'wrr',
|
79
|
+
:nodes => [
|
80
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
81
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
82
|
+
node,
|
83
|
+
],
|
84
|
+
)
|
85
|
+
|
86
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
87
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([])
|
88
|
+
cluster.start_monitoring!
|
89
|
+
|
90
|
+
@stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'does not start interpol nodes when the remote regular cluster is non-existent' do
|
94
|
+
node = Factory.node(:interpol => true, :address => '127.0.0.3')
|
95
|
+
cluster = Factory.active_active_cluster(
|
96
|
+
:fwmark => 100,
|
97
|
+
:scheduler => 'wrr',
|
98
|
+
:nodes => [
|
99
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
100
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
101
|
+
node,
|
102
|
+
],
|
103
|
+
)
|
104
|
+
|
105
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
106
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([])
|
107
|
+
cluster.start_monitoring!
|
108
|
+
|
109
|
+
@stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
|
110
|
+
end
|
111
|
+
|
112
|
+
it 'starts all interpol nodes if both the regular and relay remote clusters exist' do
|
113
|
+
node = Factory.node(:interpol => true, :address => '127.0.0.3')
|
114
|
+
cluster = Factory.active_active_cluster(
|
115
|
+
:fwmark => 100,
|
116
|
+
:scheduler => 'wrr',
|
117
|
+
:nodes => [
|
118
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
119
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
120
|
+
node,
|
121
|
+
],
|
122
|
+
)
|
123
|
+
|
124
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
125
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
126
|
+
cluster.start_monitoring!
|
127
|
+
cluster.monitor_nodes
|
128
|
+
|
129
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
|
130
|
+
end
|
131
|
+
|
132
|
+
it 'does not start interpol nodes if their "lb_source_location" is included in the "non_egress_locations"' do
|
133
|
+
node = Factory.node(:interpol => true, :address => '127.0.0.3')
|
134
|
+
cluster = Factory.active_active_cluster(
|
135
|
+
:fwmark => 100,
|
136
|
+
:scheduler => 'wrr',
|
137
|
+
:non_egress_locations => ['test'],
|
138
|
+
:nodes => [
|
139
|
+
Factory.node(:interpol => false, :address => '127.0.0.1'),
|
140
|
+
Factory.node(:interpol => false, :address => '127.0.0.2'),
|
141
|
+
node,
|
142
|
+
],
|
143
|
+
)
|
144
|
+
|
145
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
|
146
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
|
147
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
|
148
|
+
BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
|
149
|
+
cluster.start_monitoring!
|
150
|
+
cluster.monitor_nodes
|
151
|
+
|
152
|
+
@stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
|
153
|
+
@stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 55')
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
describe "#monitor_nodes" do
|
158
|
+
it "update weight of local ipvs nodes" do
|
159
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
160
|
+
node = Factory.node(:address => '127.0.0.1')
|
161
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
162
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
|
163
|
+
cluster.start_monitoring!
|
164
|
+
@stub_executor.commands.clear
|
165
|
+
|
166
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
167
|
+
cluster.monitor_nodes
|
168
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(41)
|
169
|
+
cluster.monitor_nodes
|
170
|
+
|
171
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
|
172
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 41")
|
173
|
+
end
|
174
|
+
|
175
|
+
it "update weight of relay ipvs nodes" do
|
176
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
177
|
+
node = Factory.node(:address => '127.0.0.1')
|
178
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
179
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :offset => 10000, :nodes => [node, interpol_node])
|
180
|
+
cluster.start_monitoring!
|
181
|
+
@stub_executor.commands.clear
|
182
|
+
|
183
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
184
|
+
cluster.monitor_nodes
|
185
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(41)
|
186
|
+
cluster.monitor_nodes
|
187
|
+
|
188
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56")
|
189
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 41")
|
190
|
+
end
|
191
|
+
|
192
|
+
it "does not update remote nodes for relay fwmark" do
|
193
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 91,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
194
|
+
node = Factory.node(:address => '127.0.0.1')
|
195
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
196
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :offset => 10_000)
|
197
|
+
@stub_executor.commands.clear
|
198
|
+
|
199
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
200
|
+
cluster.resume_monitoring!
|
201
|
+
cluster.monitor_nodes
|
202
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
|
203
|
+
cluster.monitor_nodes
|
204
|
+
|
205
|
+
@stub_executor.commands.should_not include("ipvsadm --edit-server --fwmark-service 10100 --real-server 172.27.3.1 --ipip --weight 56")
|
206
|
+
end
|
207
|
+
|
208
|
+
it "update weight of remote nodes" do
|
209
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
210
|
+
node = Factory.node(:address => '127.0.0.1')
|
211
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
212
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
|
213
|
+
cluster.start_monitoring!
|
214
|
+
@stub_executor.commands.clear
|
215
|
+
|
216
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
217
|
+
cluster.monitor_nodes
|
218
|
+
|
219
|
+
cluster.remote_nodes.first.weight.should == 45
|
220
|
+
|
221
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
|
222
|
+
cluster.monitor_nodes
|
223
|
+
|
224
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
|
225
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
|
226
|
+
end
|
227
|
+
|
228
|
+
it "does not update weight of remote nodes if the weight has not changed" do
|
229
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 55}])
|
230
|
+
node = Factory.node(:address => '127.0.0.1')
|
231
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
232
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
|
233
|
+
cluster.start_monitoring!
|
234
|
+
cluster.monitor_nodes
|
235
|
+
@stub_executor.commands.clear
|
236
|
+
|
237
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
238
|
+
cluster.monitor_nodes
|
239
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
240
|
+
cluster.monitor_nodes
|
241
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
242
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
243
|
+
cluster.monitor_nodes
|
244
|
+
|
245
|
+
@stub_executor.commands.should == ["ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45"]
|
246
|
+
end
|
247
|
+
|
248
|
+
it "update weight of remote not returned to 0" do
|
249
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
250
|
+
node = Factory.node(:address => '127.0.0.1')
|
251
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
252
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 2)
|
253
|
+
cluster.start_monitoring!
|
254
|
+
@stub_executor.commands.clear
|
255
|
+
|
256
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
257
|
+
cluster.monitor_nodes
|
258
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
259
|
+
cluster.monitor_nodes
|
260
|
+
|
261
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
|
262
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 0")
|
263
|
+
end
|
264
|
+
|
265
|
+
it "adds newly discovered remote nodes to ipvs" do
|
266
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
267
|
+
node = Factory.node(:address => '127.0.0.1')
|
268
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
269
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
|
270
|
+
cluster.start_monitoring!
|
271
|
+
cluster.instance_variable_get(:@remote_nodes).size.should == 0
|
272
|
+
|
273
|
+
@stub_executor.commands.clear
|
274
|
+
|
275
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
276
|
+
BigBrother::HealthFetcher.stub(:current_health).and_return(56)
|
277
|
+
cluster.monitor_nodes
|
278
|
+
|
279
|
+
cluster.remote_nodes.size.should == 1
|
280
|
+
|
281
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return(
|
282
|
+
[
|
283
|
+
{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65},
|
284
|
+
{'aggregated_health' => 100,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 40},
|
285
|
+
]
|
286
|
+
)
|
287
|
+
cluster.monitor_nodes
|
288
|
+
|
289
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
|
290
|
+
@stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45")
|
291
|
+
@stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
|
292
|
+
@stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 40")
|
293
|
+
cluster.instance_variable_get(:@remote_nodes).size.should == 2
|
294
|
+
end
|
295
|
+
|
296
|
+
it "removes a remote node from ipvs if it has been down for too many ticks" do
|
297
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
298
|
+
node = Factory.node(:address => '127.0.0.1')
|
299
|
+
interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
|
300
|
+
cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 100)
|
301
|
+
cluster.start_monitoring!
|
302
|
+
cluster.monitor_nodes
|
303
|
+
@stub_executor.commands.clear
|
304
|
+
|
305
|
+
(cluster.max_down_ticks + 1).times do
|
306
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
|
307
|
+
cluster.monitor_nodes
|
308
|
+
end
|
309
|
+
|
310
|
+
@stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 100 --real-server 172.27.3.1")
|
311
|
+
cluster.instance_variable_get(:@remote_nodes).should be_empty
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
describe "#synchronize!" do
|
316
|
+
it "does not remove remote nodes" do
|
317
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
|
318
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
|
319
|
+
cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
|
320
|
+
|
321
|
+
cluster.synchronize!
|
322
|
+
|
323
|
+
@stub_executor.commands.should be_empty
|
324
|
+
end
|
325
|
+
|
326
|
+
it "does not append remote_nodes to nodes" do
|
327
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
|
328
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
|
329
|
+
cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
|
330
|
+
|
331
|
+
cluster.synchronize!
|
332
|
+
|
333
|
+
cluster.nodes.size.should == 1
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
describe "synchronize!" do
|
338
|
+
it "continues to monitor clusters that were already monitored" do
|
339
|
+
BigBrother.ipvs.stub(:running_configuration).and_return('1' => ['127.0.0.1'])
|
340
|
+
cluster = Factory.active_active_cluster(:fwmark => 1)
|
341
|
+
|
342
|
+
cluster.synchronize!
|
343
|
+
|
344
|
+
cluster.should be_monitored
|
345
|
+
end
|
346
|
+
|
347
|
+
it "does not monitor clusters that were already monitored" do
|
348
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({})
|
349
|
+
cluster = Factory.active_active_cluster(:fwmark => 1)
|
350
|
+
|
351
|
+
cluster.synchronize!
|
352
|
+
|
353
|
+
cluster.should_not be_monitored
|
354
|
+
end
|
355
|
+
|
356
|
+
it "does not attempt to re-add the services it was monitoring" do
|
357
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
|
358
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
|
359
|
+
|
360
|
+
cluster.synchronize!
|
361
|
+
|
362
|
+
@stub_executor.commands.should be_empty
|
363
|
+
end
|
364
|
+
|
365
|
+
it "removes relay nodes that are no longer part of the cluster" do
|
366
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '127.0.1.1']})
|
367
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1')])
|
368
|
+
|
369
|
+
cluster.synchronize!
|
370
|
+
|
371
|
+
@stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1")
|
372
|
+
@stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1")
|
373
|
+
end
|
374
|
+
|
375
|
+
it "adds new relay nodes to the cluster" do
|
376
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
|
377
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1")])
|
378
|
+
|
379
|
+
cluster.synchronize!
|
380
|
+
|
381
|
+
@stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 0")
|
382
|
+
@stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 0")
|
383
|
+
end
|
384
|
+
|
385
|
+
it "does not add remote nodes to relay ipvs fwmark" do
|
386
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
|
387
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
|
388
|
+
BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
|
389
|
+
|
390
|
+
cluster.start_monitoring!
|
391
|
+
cluster.synchronize!
|
392
|
+
|
393
|
+
@stub_executor.commands.should_not include("ipvsadm --add-server --fwmark-service 10001 --real-server 172.27.3.1 --ipip --weight 0")
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
397
|
+
describe "#stop_monitoring!" do
|
398
|
+
it "deletes the relay fwmark" do
|
399
|
+
cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [])
|
400
|
+
cluster.stop_monitoring!
|
401
|
+
|
402
|
+
@stub_executor.commands.should include("ipvsadm --delete-service --fwmark-service 10001")
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
describe "#incorporate_state" do
|
407
|
+
it "starts a relay_fwmark when it is not started" do
|
408
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
|
409
|
+
active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
|
410
|
+
cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
|
411
|
+
|
412
|
+
active_active_cluster.incorporate_state(cluster)
|
413
|
+
|
414
|
+
@stub_executor.commands.should == ["ipvsadm --add-service --fwmark-service 10001 --scheduler wrr", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.0.1 --ipip --weight 80", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 100"]
|
415
|
+
end
|
416
|
+
|
417
|
+
it "does not starts a relay_fwmark when the cluster is not running" do
|
418
|
+
BigBrother.ipvs.stub(:running_configuration).and_return({'2' => ['127.0.0.1']})
|
419
|
+
active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
|
420
|
+
cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
|
421
|
+
|
422
|
+
active_active_cluster.incorporate_state(cluster)
|
423
|
+
|
424
|
+
@stub_executor.commands.should be_empty
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
describe "#stop_relay_fwmark" do
|
429
|
+
it "stops relay fwmark and all nodes in the relay fwmark" do
|
430
|
+
active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
|
431
|
+
|
432
|
+
active_active_cluster.stop_relay_fwmark
|
433
|
+
|
434
|
+
@stub_executor.commands.should == ["ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.0.1", "ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1", "ipvsadm --delete-service --fwmark-service 10001"]
|
435
|
+
end
|
436
|
+
end
|
437
|
+
end
|