who_can 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +7 -0
- data/.rspec +4 -0
- data/.rvmrc +1 -0
- data/Gemfile +8 -0
- data/Rakefile +2 -0
- data/lib/who_can.rb +64 -0
- data/lib/who_can/base.rb +72 -0
- data/lib/who_can/connection_manager.rb +268 -0
- data/lib/who_can/connection_wrapper.rb +210 -0
- data/lib/who_can/heartbeater.rb +7 -0
- data/lib/who_can/heartbeater/beat.rb +72 -0
- data/lib/who_can/heartbeater/ekg.rb +169 -0
- data/lib/who_can/logging.rb +12 -0
- data/lib/who_can/pinger.rb +82 -0
- data/lib/who_can/responder.rb +129 -0
- data/lib/who_can/version.rb +3 -0
- data/poc/heartbeater.rb +37 -0
- data/poc/submitter.rb +180 -0
- data/poc/who_can_with_hearbeat.rb +20 -0
- data/scripts/cross_cluster_ping.rb +107 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/support/evented_spec_extensions.rb +14 -0
- data/spec/support/logging.rb +8 -0
- data/spec/support/logging_progress_bar_formatter.rb +14 -0
- data/spec/who_can/base_spec.rb +66 -0
- data/spec/who_can/connection_manager_spec.rb +260 -0
- data/spec/who_can/connection_wrapper_spec.rb +91 -0
- data/spec/who_can/heartbeater/beat_spec.rb +101 -0
- data/spec/who_can/heartbeater/ekg_spec.rb +45 -0
- data/spec/who_can/pinger_responder_integration_spec.rb +63 -0
- data/spec/who_can/pinger_spec.rb +82 -0
- data/spec/who_can/responder_spec.rb +48 -0
- data/who_can.gemspec +32 -0
- metadata +290 -0
@@ -0,0 +1,260 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module WhoCan
|
4
|
+
describe 'ConnectionManager' do
|
5
|
+
include EventedSpec::EMSpec
|
6
|
+
default_timeout 3.0
|
7
|
+
|
8
|
+
em_before do
|
9
|
+
@connection1 = ConnectionWrapper.new("amqp://localhost")
|
10
|
+
@connection2 = ConnectionWrapper.new("amqp://localhost")
|
11
|
+
@connection_manager = ConnectionManager.new(@connection1, @connection2)
|
12
|
+
end
|
13
|
+
|
14
|
+
def clean_and_done
|
15
|
+
# logger.debug {"calling test done on next tick"}
|
16
|
+
EM.next_tick do
|
17
|
+
# logger.debug {"test done, closing"}
|
18
|
+
@connection_manager.close do
|
19
|
+
done
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def neuter_connection(connection)
|
25
|
+
flexmock(connection).should_receive(:connect).and_return true
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
describe "under totally normal circumstances" do
|
30
|
+
|
31
|
+
it "should enter a usable state after calling start" do
|
32
|
+
@connection_manager.start do
|
33
|
+
# logger.debug {"we DID get the start callback"}
|
34
|
+
@connection_manager.should be_usable
|
35
|
+
clean_and_done
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should enter the degraded state if #2 never starts" do
|
40
|
+
neuter_connection(@connection2)
|
41
|
+
@connection_manager.start do
|
42
|
+
# logger.debug {"we DID get the start callback for #2 not starting"}
|
43
|
+
@connection_manager.should be_degraded
|
44
|
+
clean_and_done
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should be useable after grace period if only secondary starts up" do
|
49
|
+
neuter_connection(@connection1)
|
50
|
+
@connection_manager.primary_grace_time = 0.1
|
51
|
+
@connection_manager.start do
|
52
|
+
# logger.debug {"we DID get the start callback for primary not starting"}
|
53
|
+
|
54
|
+
@connection_manager.should be_failed_over
|
55
|
+
@connection_manager.should be_usable
|
56
|
+
clean_and_done
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
describe "primary failing" do
|
63
|
+
it "should get the 2nd connection after the first goes down and does not come back and #2 is open" do
|
64
|
+
@connection_manager.on_failure do
|
65
|
+
neuter_connection(@connection1)
|
66
|
+
@connection_manager.on_open do
|
67
|
+
@connection_manager.connection.should == @connection2
|
68
|
+
@connection_manager.should be_failed_over
|
69
|
+
clean_and_done
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
@connection_manager.start do
|
74
|
+
@connection2.on_open do
|
75
|
+
@connection_manager.should be_usable
|
76
|
+
@connection1.ekg.heartbeat_failure!
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should failback to primary when it comes back up" do
|
82
|
+
@connection_manager.on_failure do
|
83
|
+
#neuter_connection(@connection1)
|
84
|
+
@connection_manager.on_open do
|
85
|
+
@connection_manager.connection.should == @connection2
|
86
|
+
@connection_manager.should be_failed_over
|
87
|
+
|
88
|
+
#this 2nd on_failure will be triggered by the primary becoming available
|
89
|
+
@connection_manager.on_failure do
|
90
|
+
@connection_manager.on_open do
|
91
|
+
@connection_manager.should be_connected
|
92
|
+
@connection_manager.connection.should == @connection1
|
93
|
+
clean_and_done
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
@connection_manager.start do
|
100
|
+
@connection2.on_open do
|
101
|
+
@connection_manager.should be_usable
|
102
|
+
@connection1.ekg.heartbeat_failure!
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should go from degraded to down if primary was up but secondary was not" do
|
108
|
+
neuter_connection(@connection2)
|
109
|
+
@connection_manager.on_failure do
|
110
|
+
@connection_manager.should be_down
|
111
|
+
clean_and_done
|
112
|
+
end
|
113
|
+
|
114
|
+
@connection_manager.start do
|
115
|
+
neuter_connection(@connection1)
|
116
|
+
@connection1.ekg.heartbeat_failure!
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
|
123
|
+
describe "secondary going down" do
|
124
|
+
|
125
|
+
it "should be degraded when down" do
|
126
|
+
@connection_manager.start do
|
127
|
+
@connection2.on_open do
|
128
|
+
@connection_manager.should be_connected
|
129
|
+
|
130
|
+
@connection2.on_failure do
|
131
|
+
#make sure #2 never comes back up (seriously, that's gross)
|
132
|
+
neuter_connection(@connection2)
|
133
|
+
EM.next_tick do
|
134
|
+
@connection_manager.should be_degraded
|
135
|
+
clean_and_done
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
#then fail #2
|
140
|
+
@connection2.ekg.heartbeat_failure!
|
141
|
+
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
it "should become connected again when it comes back" do
|
147
|
+
@connection_manager.start do
|
148
|
+
@connection2.on_open do
|
149
|
+
@connection_manager.should be_connected
|
150
|
+
|
151
|
+
@connection2.on_failure do
|
152
|
+
# logger.debug {"connection2 failure"}
|
153
|
+
#we failed #2, now wait for it to come back up
|
154
|
+
@connection2.on_open do
|
155
|
+
#it's back up - now make sure we're connected
|
156
|
+
EM.next_tick do
|
157
|
+
@connection_manager.should be_connected
|
158
|
+
clean_and_done
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
# logger.debug {"heartbeat failure"}
|
163
|
+
@connection2.ekg.heartbeat_failure!
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
|
170
|
+
end
|
171
|
+
|
172
|
+
describe "both failing" do
|
173
|
+
it "should enter a failed over state when secondary comes back up" do
|
174
|
+
@connection_manager.on_failure do
|
175
|
+
neuter_connection(@connection1)
|
176
|
+
@connection_manager.should be_failed_over
|
177
|
+
#second failure is from number 2 going down
|
178
|
+
@connection_manager.on_failure do
|
179
|
+
@connection_manager.should be_down
|
180
|
+
#now #2 comes back up
|
181
|
+
@connection_manager.on_open do
|
182
|
+
@connection_manager.should be_failed_over
|
183
|
+
@connection_manager.connection.should == @connection2
|
184
|
+
clean_and_done
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
@connection_manager.start do
|
190
|
+
@connection2.on_open do
|
191
|
+
@connection_manager.should be_usable
|
192
|
+
@connection1.ekg.heartbeat_failure!
|
193
|
+
EM.next_tick { @connection2.ekg.heartbeat_failure! }
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
|
199
|
+
it "should come back up degraded when primary comes back up" do
|
200
|
+
@connection_manager.on_failure do
|
201
|
+
neuter_connection(@connection2)
|
202
|
+
@connection_manager.should be_failed_over
|
203
|
+
#second failure is from number 2 going down
|
204
|
+
@connection_manager.on_failure do
|
205
|
+
@connection_manager.should be_down
|
206
|
+
#now #1 comes back up
|
207
|
+
@connection_manager.on_open do
|
208
|
+
@connection_manager.should be_degraded
|
209
|
+
@connection_manager.connection.should == @connection1
|
210
|
+
clean_and_done
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
@connection_manager.start do
|
216
|
+
@connection2.on_open do
|
217
|
+
@connection_manager.should be_usable
|
218
|
+
@connection1.ekg.heartbeat_failure!
|
219
|
+
EM.next_tick { @connection2.ekg.heartbeat_failure! }
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
end
|
224
|
+
|
225
|
+
|
226
|
+
end
|
227
|
+
|
228
|
+
|
229
|
+
#
|
230
|
+
# describe 'connect!' do
|
231
|
+
# it %[should take a block that will be called back when a connection is ready]
|
232
|
+
# it %[should fire the on_open deferred when ready]
|
233
|
+
# it %[should reset the on_open deferred after it's first fired]
|
234
|
+
#
|
235
|
+
# describe 'primary down' do
|
236
|
+
# it %[should time out waiting and use the secondary if the secondary is up]
|
237
|
+
# end
|
238
|
+
#
|
239
|
+
# describe 'secondary down' do
|
240
|
+
# it %[should not affect startup]
|
241
|
+
# end
|
242
|
+
# end
|
243
|
+
#
|
244
|
+
# describe 'failure' do
|
245
|
+
# describe 'of the primary connection' do
|
246
|
+
# it %[should fire the on_failure deferred]
|
247
|
+
# it %[should fire the on_open deferred when the secondary is ready]
|
248
|
+
# end
|
249
|
+
#
|
250
|
+
# describe 'of the secondary connection' do
|
251
|
+
# it %[should not fire the on_failure deferred]
|
252
|
+
# end
|
253
|
+
# end
|
254
|
+
#
|
255
|
+
# describe 'fail-back' do
|
256
|
+
# end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module WhoCan
|
4
|
+
describe :ConnectionWrapper do
|
5
|
+
|
6
|
+
|
7
|
+
before do
|
8
|
+
@config = 'amqp://localhost'
|
9
|
+
@cw = ConnectionWrapper.new(@config, :reconnect_delay => 0.2)
|
10
|
+
# flexmock(AMQP).should_receive(:connect).with(@config, Hash, Proc)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe :state_machine do
|
14
|
+
it %[should be in state new] do
|
15
|
+
@cw.new?.should be_true
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
describe :connect do
|
21
|
+
include EventedSpec::EMSpec
|
22
|
+
default_timeout 2.0
|
23
|
+
|
24
|
+
def with_connection
|
25
|
+
@cw.connect do
|
26
|
+
# logger.debug { "on_open fired" }
|
27
|
+
yield
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def close_and_done
|
32
|
+
# logger.debug { "calling @cw.close!" }
|
33
|
+
@cw.close! { done }
|
34
|
+
end
|
35
|
+
|
36
|
+
describe 'success' do
|
37
|
+
it %[should connect and fire the callbacks] do
|
38
|
+
with_connection do
|
39
|
+
@cw.should be_connected
|
40
|
+
@cw.can_close?.should be_true
|
41
|
+
@cw.amqp_session.should_not be_nil
|
42
|
+
@cw.amqp_session.should be_kind_of(AMQP::Session)
|
43
|
+
@cw.amqp_session.should be_connected
|
44
|
+
|
45
|
+
close_and_done
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe 'initial connection failure' do
|
51
|
+
before do
|
52
|
+
flexmock(AMQP).should_receive(:connect).with(@config, Hash, Proc).and_return do |conf,opts,blk|
|
53
|
+
# logger.debug { "firing on_tcp_connection_failure callback" }
|
54
|
+
opts[:on_tcp_connection_failure].call(@config)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
it %[should enter retrying state] do
|
59
|
+
@cw.on_initial_connection_failure do
|
60
|
+
# logger.debug { "initial connection failed" }
|
61
|
+
|
62
|
+
@cw.should be_retrying
|
63
|
+
|
64
|
+
@cw.close! { done }
|
65
|
+
end
|
66
|
+
|
67
|
+
@cw.connect
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe 'established connection failure' do
|
72
|
+
it %[should enter retrying state] do
|
73
|
+
orig_on_failure = @cw.on_failure
|
74
|
+
|
75
|
+
with_connection do
|
76
|
+
@cw.on_failure do
|
77
|
+
orig_on_failure.should_not == @cw.on_failure
|
78
|
+
@cw.should be_retrying
|
79
|
+
|
80
|
+
close_and_done
|
81
|
+
end
|
82
|
+
|
83
|
+
# logger.debug { "calling heartbeat_failure!" }
|
84
|
+
@cw.ekg.heartbeat_failure!
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module WhoCan::Heartbeater
|
4
|
+
describe 'Heartbeater' do
|
5
|
+
include EventedSpec::AMQPSpec
|
6
|
+
default_timeout 3.0
|
7
|
+
default_options :host => 'localhost'
|
8
|
+
|
9
|
+
describe 'Beat' do
|
10
|
+
before do
|
11
|
+
@beat_id = "beat_0"
|
12
|
+
end
|
13
|
+
|
14
|
+
amqp_before do
|
15
|
+
@chan = AMQP::Channel.new(AMQP.connection)
|
16
|
+
@chan.once_opened do
|
17
|
+
@queue = @chan.queue('', :exclusive => true, :auto_delete => true)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
it %[should fire its callback on success] do
|
22
|
+
@chan.once_opened do
|
23
|
+
@chan.should be_open
|
24
|
+
|
25
|
+
beat = Beat.new(@chan, @queue, @beat_id, 0.5)
|
26
|
+
beat.should be_pending
|
27
|
+
|
28
|
+
beat.callback { done }
|
29
|
+
|
30
|
+
beat.start!
|
31
|
+
|
32
|
+
beat.after_publish do
|
33
|
+
beat.ping_received!
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
it %[should fire its errback if it times out] do
|
39
|
+
@chan.once_opened do
|
40
|
+
@default_exchange = flexmock(:default_exchange) do |m|
|
41
|
+
m.should_receive(:publish).once.with(String, Hash, Proc).and_return do |_,_,blk|
|
42
|
+
blk.call
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
mchan = flexmock(@chan, :default_exchange => @default_exchange)
|
47
|
+
|
48
|
+
beat = Beat.new(mchan, @queue, @beat_id, 0.25)
|
49
|
+
|
50
|
+
beat.errback do
|
51
|
+
# logger.debug { "errback fired!" }
|
52
|
+
done
|
53
|
+
end
|
54
|
+
|
55
|
+
beat.callback do
|
56
|
+
# logger.debug { "GAH! callback fired! this is bad!" }
|
57
|
+
raise "NO! FAIL!"
|
58
|
+
end
|
59
|
+
|
60
|
+
beat.start!
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
it %[should be cancelled? after cancel!] do
|
65
|
+
@chan.once_opened do
|
66
|
+
beat = Beat.new(@chan, @queue, @beat_id, 0.5)
|
67
|
+
|
68
|
+
beat.callback do
|
69
|
+
# logger.debug { "I HAVE BEEN CALLED BACK!!" }
|
70
|
+
beat.should be_cancelled
|
71
|
+
done
|
72
|
+
end
|
73
|
+
|
74
|
+
beat.after_publish do
|
75
|
+
beat.cancel!
|
76
|
+
end
|
77
|
+
|
78
|
+
beat.start!
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
it "should call the errback on a queue opening failure" do
|
83
|
+
@chan.once_opened do
|
84
|
+
beat = Beat.new(@chan, @queue, @beat_id, 0.25)
|
85
|
+
flexmock(@chan).should_receive(:queue).and_raise(AMQ::Client::ConnectionClosedError)
|
86
|
+
|
87
|
+
beat.errback do
|
88
|
+
# logger.debug { "errback fired!" }
|
89
|
+
done
|
90
|
+
end
|
91
|
+
|
92
|
+
beat.callback do
|
93
|
+
raise "FAIL!"
|
94
|
+
end
|
95
|
+
|
96
|
+
beat.start!
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|