big_brother 0.6.8 → 0.8.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,5 @@
1
+ require 'json'
2
+
1
3
  module BigBrother
2
4
  class HealthFetcher
3
5
  def self.current_health(address, port, path)
@@ -8,6 +10,16 @@ module BigBrother
8
10
  response.response_header.status == 200 ? _parse_health(response) : 0
9
11
  end
10
12
 
13
+ def self.interpol_status(interpol_node, fwmark)
14
+ url = "http://#{interpol_node.address}:#{interpol_node.port}/fwmark/#{fwmark}"
15
+
16
+ BigBrother.logger.debug("Fetching health from #{url}")
17
+ response = EventMachine::HttpRequest.new(url).get
18
+ response.response_header.status == 200 ? JSON.parse(response.response) : []
19
+ rescue JSON::ParserError
20
+ []
21
+ end
22
+
11
23
  def self._parse_health(response)
12
24
  if response.response_header.has_key?('X_HEALTH')
13
25
  response.response_header['X_HEALTH'].to_i
@@ -2,7 +2,9 @@ require 'net/http'
2
2
 
3
3
  module BigBrother
4
4
  class Node
5
- attr_reader :address, :port, :path, :start_time, :weight
5
+ attr_reader :address, :port, :path, :start_time, :priority, :max_weight
6
+ attr_accessor :weight, :down_tick_count
7
+ INITIAL_WEIGHT = 1
6
8
 
7
9
  def initialize(attributes={})
8
10
  @address = attributes[:address]
@@ -10,6 +12,10 @@ module BigBrother
10
12
  @path = attributes[:path]
11
13
  @weight = attributes[:weight]
12
14
  @start_time = attributes.fetch(:start_time, Time.now.to_i)
15
+ @priority = attributes.fetch(:priority, 0)
16
+ @interpol = attributes.fetch(:interpol, false)
17
+ @max_weight = attributes[:max_weight]
18
+ @down_tick_count = 0
13
19
  end
14
20
 
15
21
  def age
@@ -27,26 +33,45 @@ module BigBrother
27
33
  @weight = nil
28
34
  end
29
35
 
30
- def monitor(cluster)
31
- new_weight = _determine_weight(cluster)
32
- return unless cluster.monitored?
33
- if new_weight != @weight
34
- BigBrother.ipvs.edit_node(cluster.fwmark, address, new_weight)
35
- @weight = new_weight
36
- end
36
+ def interpol?
37
+ @interpol
37
38
  end
38
39
 
39
40
  def ==(other)
40
41
  address == other.address && port == other.port
41
42
  end
43
+ alias eql? ==
44
+
45
+ def hash
46
+ [@address, @port].hash
47
+ end
48
+
49
+ def <=>(other)
50
+ return 1 if self.weight.to_i.zero?
51
+ return -1 if other.weight.to_i.zero?
52
+ comparison = self.priority <=> other.priority
53
+ if comparison.zero?
54
+ self.address <=> other.address
55
+ else
56
+ comparison
57
+ end
58
+ end
42
59
 
43
- def _determine_weight(cluster)
60
+ def monitor(cluster)
44
61
  if cluster.up_file_exists?
45
62
  100
46
63
  elsif cluster.down_file_exists?
47
64
  0
48
65
  else
49
- _weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time)
66
+ _cap_weight(_weight_health(BigBrother::HealthFetcher.current_health(@address, @port, @path), cluster.ramp_up_time))
67
+ end
68
+ end
69
+
70
+ def _cap_weight(health)
71
+ if !@max_weight.nil? && @max_weight.is_a?(Integer) && @max_weight > 0 && @max_weight < health
72
+ @max_weight
73
+ else
74
+ health
50
75
  end
51
76
  end
52
77
 
@@ -1,3 +1,3 @@
1
1
  module BigBrother
2
- VERSION = "0.6.8"
2
+ VERSION = "0.8.7"
3
3
  end
@@ -0,0 +1,92 @@
1
+ type: map
2
+ mapping:
3
+ "_big_brother":
4
+ type: map
5
+ mapping:
6
+ "offset":
7
+ type: int
8
+ "check_interval":
9
+ type: int
10
+ "max_down_ticks":
11
+ type: int
12
+ "backend_mode":
13
+ type: str
14
+ enum: [active_passive, active_active]
15
+ "scheduler":
16
+ type: str
17
+ enum: [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
18
+ "ramp_up_time":
19
+ type: int
20
+ "has_downpage":
21
+ type: bool
22
+ "nagios":
23
+ type: map
24
+ mapping:
25
+ "server":
26
+ type: str
27
+ "check":
28
+ type: str
29
+ "host":
30
+ type: str
31
+ "clusters":
32
+ type: seq
33
+ required: yes
34
+ sequence:
35
+ - type: map
36
+ mapping:
37
+ "non_egress_locations":
38
+ type: seq
39
+ sequence:
40
+ - type: str
41
+ "cluster_name":
42
+ type: str
43
+ required: yes
44
+ "check_interval":
45
+ type: int
46
+ "max_down_ticks":
47
+ type: int
48
+ "backend_mode":
49
+ type: str
50
+ enum: [active_passive, active_active]
51
+ "scheduler":
52
+ type: str
53
+ enum: [rr, wrr, lc, wlc, lblc, lblcr, dh, sh, sed, nq]
54
+ required: yes
55
+ "fwmark":
56
+ type: int
57
+ required: yes
58
+ "ramp_up_time":
59
+ type: int
60
+ "has_downpage":
61
+ type: bool
62
+ "nagios":
63
+ type: map
64
+ mapping:
65
+ "server":
66
+ type: str
67
+ "check":
68
+ type: str
69
+ "host":
70
+ type: str
71
+ "nodes":
72
+ type: seq
73
+ required: yes
74
+ sequence:
75
+ - type: map
76
+ mapping:
77
+ "address":
78
+ type: str
79
+ pattern: /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
80
+ required: yes
81
+ "interpol":
82
+ type: bool
83
+ "max_weight":
84
+ type: int
85
+ "priority":
86
+ type: int
87
+ "port":
88
+ type: int
89
+ range: { max: 65535, min: 1 }
90
+ "path":
91
+ type: str
92
+ pattern: /^\//
@@ -0,0 +1,437 @@
1
+ require 'spec_helper'
2
+
3
+ describe BigBrother::ActiveActiveCluster do
4
+ before { BigBrother::HealthFetcher.stub(:current_health).and_return(10) }
5
+
6
+ describe '#start_monitoring!' do
7
+ it 'starts all non interpol nodes' do
8
+ cluster = Factory.active_active_cluster(
9
+ :fwmark => 100,
10
+ :scheduler => 'wrr',
11
+ :nodes => [
12
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
13
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
14
+ Factory.node(:interpol => true, :address => '127.0.0.3'),
15
+ ],
16
+ )
17
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
18
+ cluster.start_monitoring!
19
+
20
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 1')
21
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.2 --ipip --weight 1')
22
+ @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 127.0.0.3 --ipip --weight 1')
23
+ end
24
+
25
+ it 'starts all relay fwmark service' do
26
+ cluster = Factory.active_active_cluster(
27
+ :fwmark => 100,
28
+ :scheduler => 'wrr',
29
+ :offset => 10000,
30
+ :nodes => [
31
+ Factory.node(:interpol => true, :address => '127.0.0.3'),
32
+ ],
33
+ )
34
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
35
+ cluster.start_monitoring!
36
+
37
+ @stub_executor.commands.should include('ipvsadm --add-service --fwmark-service 10100 --scheduler wrr')
38
+ end
39
+
40
+ it 'starts all local_nodes on relay fwmark' do
41
+ cluster = Factory.active_active_cluster(
42
+ :fwmark => 100,
43
+ :scheduler => 'wrr',
44
+ :offset => 10000,
45
+ :nodes => [
46
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
47
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
48
+ Factory.node(:interpol => true, :address => '127.0.0.3'),
49
+ ],
50
+ )
51
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
52
+ cluster.start_monitoring!
53
+
54
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 1')
55
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.2 --ipip --weight 1')
56
+ @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 10100 --real-server 127.0.0.3 --ipip --weight 1')
57
+ end
58
+
59
+ it 'does not attempt to retrieve interpol status if there is no interpol node' do
60
+ cluster = Factory.active_active_cluster(
61
+ :fwmark => 100,
62
+ :scheduler => 'wrr',
63
+ :nodes => [
64
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
65
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
66
+ ],
67
+ )
68
+
69
+ BigBrother::HealthFetcher.should_not_receive(:interpol_status)
70
+
71
+ cluster.start_monitoring!
72
+ end
73
+
74
+ it 'does not start interpol nodes when the remote relay cluster is non-existent' do
75
+ node = Factory.node(:interpol => true, :address => '127.0.0.3')
76
+ cluster = Factory.active_active_cluster(
77
+ :fwmark => 100,
78
+ :scheduler => 'wrr',
79
+ :nodes => [
80
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
81
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
82
+ node,
83
+ ],
84
+ )
85
+
86
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
87
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([])
88
+ cluster.start_monitoring!
89
+
90
+ @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
91
+ end
92
+
93
+ it 'does not start interpol nodes when the remote regular cluster is non-existent' do
94
+ node = Factory.node(:interpol => true, :address => '127.0.0.3')
95
+ cluster = Factory.active_active_cluster(
96
+ :fwmark => 100,
97
+ :scheduler => 'wrr',
98
+ :nodes => [
99
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
100
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
101
+ node,
102
+ ],
103
+ )
104
+
105
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
106
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([])
107
+ cluster.start_monitoring!
108
+
109
+ @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
110
+ end
111
+
112
+ it 'starts all interpol nodes if both the regular and relay remote clusters exist' do
113
+ node = Factory.node(:interpol => true, :address => '127.0.0.3')
114
+ cluster = Factory.active_active_cluster(
115
+ :fwmark => 100,
116
+ :scheduler => 'wrr',
117
+ :nodes => [
118
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
119
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
120
+ node,
121
+ ],
122
+ )
123
+
124
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
125
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
126
+ cluster.start_monitoring!
127
+ cluster.monitor_nodes
128
+
129
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
130
+ end
131
+
132
+ it 'does not start interpol nodes if their "lb_source_location" is included in the "non_egress_locations"' do
133
+ node = Factory.node(:interpol => true, :address => '127.0.0.3')
134
+ cluster = Factory.active_active_cluster(
135
+ :fwmark => 100,
136
+ :scheduler => 'wrr',
137
+ :non_egress_locations => ['test'],
138
+ :nodes => [
139
+ Factory.node(:interpol => false, :address => '127.0.0.1'),
140
+ Factory.node(:interpol => false, :address => '127.0.0.2'),
141
+ node,
142
+ ],
143
+ )
144
+
145
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
146
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45, 'lb_source_location' => 'test'}])
147
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
148
+ BigBrother::HealthFetcher.stub(:interpol_status).with(node, 10100).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 55, 'lb_source_location' => 'foo'}])
149
+ cluster.start_monitoring!
150
+ cluster.monitor_nodes
151
+
152
+ @stub_executor.commands.should_not include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45')
153
+ @stub_executor.commands.should include('ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 55')
154
+ end
155
+ end
156
+
157
+ describe "#monitor_nodes" do
158
+ it "update weight of local ipvs nodes" do
159
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
160
+ node = Factory.node(:address => '127.0.0.1')
161
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
162
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
163
+ cluster.start_monitoring!
164
+ @stub_executor.commands.clear
165
+
166
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
167
+ cluster.monitor_nodes
168
+ BigBrother::HealthFetcher.stub(:current_health).and_return(41)
169
+ cluster.monitor_nodes
170
+
171
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
172
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 41")
173
+ end
174
+
175
+ it "update weight of relay ipvs nodes" do
176
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
177
+ node = Factory.node(:address => '127.0.0.1')
178
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
179
+ cluster = Factory.active_active_cluster(:fwmark => 100, :offset => 10000, :nodes => [node, interpol_node])
180
+ cluster.start_monitoring!
181
+ @stub_executor.commands.clear
182
+
183
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
184
+ cluster.monitor_nodes
185
+ BigBrother::HealthFetcher.stub(:current_health).and_return(41)
186
+ cluster.monitor_nodes
187
+
188
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56")
189
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 41")
190
+ end
191
+
192
+ it "does not update remote nodes for relay fwmark" do
193
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 91,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
194
+ node = Factory.node(:address => '127.0.0.1')
195
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
196
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :offset => 10_000)
197
+ @stub_executor.commands.clear
198
+
199
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
200
+ cluster.resume_monitoring!
201
+ cluster.monitor_nodes
202
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
203
+ cluster.monitor_nodes
204
+
205
+ @stub_executor.commands.should_not include("ipvsadm --edit-server --fwmark-service 10100 --real-server 172.27.3.1 --ipip --weight 56")
206
+ end
207
+
208
+ it "update weight of remote nodes" do
209
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
210
+ node = Factory.node(:address => '127.0.0.1')
211
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
212
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
213
+ cluster.start_monitoring!
214
+ @stub_executor.commands.clear
215
+
216
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
217
+ cluster.monitor_nodes
218
+
219
+ cluster.remote_nodes.first.weight.should == 45
220
+
221
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65}])
222
+ cluster.monitor_nodes
223
+
224
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
225
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
226
+ end
227
+
228
+ it "does not update weight of remote nodes if the weight has not changed" do
229
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 55}])
230
+ node = Factory.node(:address => '127.0.0.1')
231
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
232
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
233
+ cluster.start_monitoring!
234
+ cluster.monitor_nodes
235
+ @stub_executor.commands.clear
236
+
237
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
238
+ cluster.monitor_nodes
239
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
240
+ cluster.monitor_nodes
241
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
242
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
243
+ cluster.monitor_nodes
244
+
245
+ @stub_executor.commands.should == ["ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 10100 --real-server 127.0.0.1 --ipip --weight 56", "ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45"]
246
+ end
247
+
248
+ it "update weight of remote not returned to 0" do
249
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
250
+ node = Factory.node(:address => '127.0.0.1')
251
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
252
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 2)
253
+ cluster.start_monitoring!
254
+ @stub_executor.commands.clear
255
+
256
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
257
+ cluster.monitor_nodes
258
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
259
+ cluster.monitor_nodes
260
+
261
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
262
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 0")
263
+ end
264
+
265
+ it "adds newly discovered remote nodes to ipvs" do
266
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
267
+ node = Factory.node(:address => '127.0.0.1')
268
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
269
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node])
270
+ cluster.start_monitoring!
271
+ cluster.instance_variable_get(:@remote_nodes).size.should == 0
272
+
273
+ @stub_executor.commands.clear
274
+
275
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
276
+ BigBrother::HealthFetcher.stub(:current_health).and_return(56)
277
+ cluster.monitor_nodes
278
+
279
+ cluster.remote_nodes.size.should == 1
280
+
281
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return(
282
+ [
283
+ {'aggregated_health' => 130,'count' => 2,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 65},
284
+ {'aggregated_health' => 100,'count' => 1,'lb_ip_address' => '172.27.3.2','lb_url' => 'http://172.27.3.2','health' => 40},
285
+ ]
286
+ )
287
+ cluster.monitor_nodes
288
+
289
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 127.0.0.1 --ipip --weight 56")
290
+ @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 45")
291
+ @stub_executor.commands.should include("ipvsadm --edit-server --fwmark-service 100 --real-server 172.27.3.1 --ipip --weight 65")
292
+ @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 100 --real-server 172.27.3.2 --ipip --weight 40")
293
+ cluster.instance_variable_get(:@remote_nodes).size.should == 2
294
+ end
295
+
296
+ it "removes a remote node from ipvs if it has been down for too many ticks" do
297
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
298
+ node = Factory.node(:address => '127.0.0.1')
299
+ interpol_node = Factory.node(:address => '172.27.3.1', :interpol => true)
300
+ cluster = Factory.active_active_cluster(:fwmark => 100, :nodes => [node, interpol_node], :max_down_ticks => 100)
301
+ cluster.start_monitoring!
302
+ cluster.monitor_nodes
303
+ @stub_executor.commands.clear
304
+
305
+ (cluster.max_down_ticks + 1).times do
306
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([])
307
+ cluster.monitor_nodes
308
+ end
309
+
310
+ @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 100 --real-server 172.27.3.1")
311
+ cluster.instance_variable_get(:@remote_nodes).should be_empty
312
+ end
313
+ end
314
+
315
+ describe "#synchronize!" do
316
+ it "does not remove remote nodes" do
317
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
318
+ cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
319
+ cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
320
+
321
+ cluster.synchronize!
322
+
323
+ @stub_executor.commands.should be_empty
324
+ end
325
+
326
+ it "does not append remote_nodes to nodes" do
327
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '172.27.1.3']})
328
+ cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
329
+ cluster.instance_variable_set(:@remote_nodes, [Factory.node(:address => '172.27.1.3')])
330
+
331
+ cluster.synchronize!
332
+
333
+ cluster.nodes.size.should == 1
334
+ end
335
+ end
336
+
337
+ describe "synchronize!" do
338
+ it "continues to monitor clusters that were already monitored" do
339
+ BigBrother.ipvs.stub(:running_configuration).and_return('1' => ['127.0.0.1'])
340
+ cluster = Factory.active_active_cluster(:fwmark => 1)
341
+
342
+ cluster.synchronize!
343
+
344
+ cluster.should be_monitored
345
+ end
346
+
347
+ it "does not monitor clusters that were already monitored" do
348
+ BigBrother.ipvs.stub(:running_configuration).and_return({})
349
+ cluster = Factory.active_active_cluster(:fwmark => 1)
350
+
351
+ cluster.synchronize!
352
+
353
+ cluster.should_not be_monitored
354
+ end
355
+
356
+ it "does not attempt to re-add the services it was monitoring" do
357
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
358
+ cluster = Factory.active_active_cluster(:fwmark => 1, :nodes => [Factory.node(:address => '127.0.0.1')])
359
+
360
+ cluster.synchronize!
361
+
362
+ @stub_executor.commands.should be_empty
363
+ end
364
+
365
+ it "removes relay nodes that are no longer part of the cluster" do
366
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1', '127.0.1.1']})
367
+ cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1')])
368
+
369
+ cluster.synchronize!
370
+
371
+ @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 1 --real-server 127.0.1.1")
372
+ @stub_executor.commands.should include("ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1")
373
+ end
374
+
375
+ it "adds new relay nodes to the cluster" do
376
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
377
+ cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1")])
378
+
379
+ cluster.synchronize!
380
+
381
+ @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 1 --real-server 127.0.1.1 --ipip --weight 0")
382
+ @stub_executor.commands.should include("ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 0")
383
+ end
384
+
385
+ it "does not add remote nodes to relay ipvs fwmark" do
386
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
387
+ cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
388
+ BigBrother::HealthFetcher.stub(:interpol_status).and_return([{'aggregated_health' => 90,'count' => 1,'lb_ip_address' => '172.27.3.1','lb_url' => 'http://172.27.3.1','health' => 45}])
389
+
390
+ cluster.start_monitoring!
391
+ cluster.synchronize!
392
+
393
+ @stub_executor.commands.should_not include("ipvsadm --add-server --fwmark-service 10001 --real-server 172.27.3.1 --ipip --weight 0")
394
+ end
395
+ end
396
+
397
+ describe "#stop_monitoring!" do
398
+ it "deletes the relay fwmark" do
399
+ cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [])
400
+ cluster.stop_monitoring!
401
+
402
+ @stub_executor.commands.should include("ipvsadm --delete-service --fwmark-service 10001")
403
+ end
404
+ end
405
+
406
+ describe "#incorporate_state" do
407
+ it "starts a relay_fwmark when it is not started" do
408
+ BigBrother.ipvs.stub(:running_configuration).and_return({'1' => ['127.0.0.1']})
409
+ active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
410
+ cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
411
+
412
+ active_active_cluster.incorporate_state(cluster)
413
+
414
+ @stub_executor.commands.should == ["ipvsadm --add-service --fwmark-service 10001 --scheduler wrr", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.0.1 --ipip --weight 80", "ipvsadm --add-server --fwmark-service 10001 --real-server 127.0.1.1 --ipip --weight 100"]
415
+ end
416
+
417
+ it "does not starts a relay_fwmark when the cluster is not running" do
418
+ BigBrother.ipvs.stub(:running_configuration).and_return({'2' => ['127.0.0.1']})
419
+ active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
420
+ cluster = Factory.cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1', :weight => 80), Factory.node(:address => "127.0.1.1", :weight => 100)])
421
+
422
+ active_active_cluster.incorporate_state(cluster)
423
+
424
+ @stub_executor.commands.should be_empty
425
+ end
426
+ end
427
+
428
+ describe "#stop_relay_fwmark" do
429
+ it "stops relay fwmark and all nodes in the relay fwmark" do
430
+ active_active_cluster = Factory.active_active_cluster(:fwmark => 1, :offset => 10000, :nodes => [Factory.node(:address => '127.0.0.1'), Factory.node(:address => "127.0.1.1"), Factory.node(:address => "127.1.1.1", :interpol => true)])
431
+
432
+ active_active_cluster.stop_relay_fwmark
433
+
434
+ @stub_executor.commands.should == ["ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.0.1", "ipvsadm --delete-server --fwmark-service 10001 --real-server 127.0.1.1", "ipvsadm --delete-service --fwmark-service 10001"]
435
+ end
436
+ end
437
+ end