who_can 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +7 -0
- data/.rspec +4 -0
- data/.rvmrc +1 -0
- data/Gemfile +8 -0
- data/Rakefile +2 -0
- data/lib/who_can.rb +64 -0
- data/lib/who_can/base.rb +72 -0
- data/lib/who_can/connection_manager.rb +268 -0
- data/lib/who_can/connection_wrapper.rb +210 -0
- data/lib/who_can/heartbeater.rb +7 -0
- data/lib/who_can/heartbeater/beat.rb +72 -0
- data/lib/who_can/heartbeater/ekg.rb +169 -0
- data/lib/who_can/logging.rb +12 -0
- data/lib/who_can/pinger.rb +82 -0
- data/lib/who_can/responder.rb +129 -0
- data/lib/who_can/version.rb +3 -0
- data/poc/heartbeater.rb +37 -0
- data/poc/submitter.rb +180 -0
- data/poc/who_can_with_hearbeat.rb +20 -0
- data/scripts/cross_cluster_ping.rb +107 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/support/evented_spec_extensions.rb +14 -0
- data/spec/support/logging.rb +8 -0
- data/spec/support/logging_progress_bar_formatter.rb +14 -0
- data/spec/who_can/base_spec.rb +66 -0
- data/spec/who_can/connection_manager_spec.rb +260 -0
- data/spec/who_can/connection_wrapper_spec.rb +91 -0
- data/spec/who_can/heartbeater/beat_spec.rb +101 -0
- data/spec/who_can/heartbeater/ekg_spec.rb +45 -0
- data/spec/who_can/pinger_responder_integration_spec.rb +63 -0
- data/spec/who_can/pinger_spec.rb +82 -0
- data/spec/who_can/responder_spec.rb +48 -0
- data/who_can.gemspec +32 -0
- metadata +290 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm 1.9.2-p290@whocan --create
|
data/Gemfile
ADDED
data/Rakefile
ADDED
data/lib/who_can.rb
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'amqp'
|
3
|
+
require 'state_machine'
|
4
|
+
require 'uuidtools'
|
5
|
+
require 'thread'
|
6
|
+
require 'monitor'
|
7
|
+
require 'logger'
|
8
|
+
require 'set'
|
9
|
+
require 'deferred'
|
10
|
+
|
11
|
+
%w[ core_ext/array/extract_options
|
12
|
+
core_ext/module/aliasing
|
13
|
+
core_ext/hash/reverse_merge
|
14
|
+
core_ext/hash/indifferent_access ].each do |p|
|
15
|
+
|
16
|
+
require "active_support/#{p}"
|
17
|
+
end
|
18
|
+
|
19
|
+
module WhoCan
|
20
|
+
DEFAULT_PING_EXCHANGE = 'who_can.default.fanout'
|
21
|
+
DEFAULT_CONNECT_URL = 'amqp://127.0.0.1'
|
22
|
+
|
23
|
+
def self.connect_url
|
24
|
+
@config ||= DEFAULT_CONNECT_URL
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.connect_url=(config)
|
28
|
+
@config = config
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.new(*a, &b)
|
32
|
+
WhoCan::Base.new(*a, &b)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.logger
|
36
|
+
@logger ||= Logger.new('/dev/null').tap { |l| l.level = Logger::FATAL }
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.logger=(log)
|
40
|
+
@logger = log
|
41
|
+
end
|
42
|
+
|
43
|
+
class WhoCanError < StandardError; end
|
44
|
+
class DelayMustBeSetError < WhoCanError
|
45
|
+
def initialize
|
46
|
+
super("delay must be set for on_ping Response object")
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
class ChannelPoolError < WhoCanError; end
|
51
|
+
class PoolIsNotOpenException < ChannelPoolError; end
|
52
|
+
class TimeoutError < WhoCanError; end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
require 'who_can/logging'
|
57
|
+
require 'who_can/base'
|
58
|
+
require 'who_can/pinger'
|
59
|
+
require 'who_can/responder'
|
60
|
+
require 'who_can/heartbeater'
|
61
|
+
require 'who_can/connection_wrapper'
|
62
|
+
require 'who_can/connection_manager'
|
63
|
+
|
64
|
+
|
data/lib/who_can/base.rb
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
module WhoCan
|
2
|
+
class Base
|
3
|
+
include Deferred::Accessors
|
4
|
+
include Logging
|
5
|
+
|
6
|
+
deferred_event :disconnection
|
7
|
+
attr_accessor :connection_opts
|
8
|
+
|
9
|
+
def initialize(connection_opts=nil)
|
10
|
+
@connection_opts = connection_opts
|
11
|
+
@connection = nil
|
12
|
+
if @connection_opts.is_a?(AMQP::Session)
|
13
|
+
@connection = @connection_opts
|
14
|
+
end
|
15
|
+
# logger.debug { "connection_opts: #{@connection_opts.inspect}" }
|
16
|
+
@responders = {}
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
def connection(&blk)
|
21
|
+
@connection ||= AMQP.connect(connection_opts || WhoCan.connect_url)
|
22
|
+
on_open(&blk) if blk
|
23
|
+
@connection
|
24
|
+
end
|
25
|
+
alias :connect! :connection
|
26
|
+
|
27
|
+
def on_open(&block)
|
28
|
+
connection.on_open(&block)
|
29
|
+
end
|
30
|
+
|
31
|
+
def start_monitoring!(opts = {})
|
32
|
+
@ekg = Heartbeater::EKG.new(connection)
|
33
|
+
@ekg.on_heartbeat_failure do
|
34
|
+
on_disconnection.succeed
|
35
|
+
close!
|
36
|
+
end
|
37
|
+
@ekg.start!
|
38
|
+
end
|
39
|
+
|
40
|
+
def new_pinger
|
41
|
+
WhoCan::Pinger.new(connection)
|
42
|
+
end
|
43
|
+
|
44
|
+
def new_responder(exchange_name)
|
45
|
+
WhoCan::Responder.new(connection, exchange_name)
|
46
|
+
end
|
47
|
+
|
48
|
+
def close!(&block)
|
49
|
+
unless @connection
|
50
|
+
block.call
|
51
|
+
return
|
52
|
+
end
|
53
|
+
cnx, @connection = @connection, nil
|
54
|
+
@channel = nil
|
55
|
+
on_disconnection(&block)
|
56
|
+
cnx.on_disconnection do
|
57
|
+
on_disconnection.succeed
|
58
|
+
end
|
59
|
+
cnx.close
|
60
|
+
on_disconnection
|
61
|
+
end
|
62
|
+
|
63
|
+
def clear_responder(ping_exchange_name)
|
64
|
+
if resp = @responders.delete(ping_exchange_name)
|
65
|
+
resp.close!
|
66
|
+
true
|
67
|
+
end
|
68
|
+
false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
@@ -0,0 +1,268 @@
|
|
1
|
+
module WhoCan
|
2
|
+
# This class handles setting up the AMQP connection. You must start the
|
3
|
+
# EventMachine reactor before running anything in this class
|
4
|
+
#
|
5
|
+
# Keeps two connections open, a primary and secondary, and reacts to changes
|
6
|
+
# in their state. In the non-failure case, when a client calls #connection on
|
7
|
+
# us, they will receive the primary connection (once it's been established).
|
8
|
+
# If the primary is down, or the connection can't be established for some
|
9
|
+
# reason, then the secondary is returned.
|
10
|
+
#
|
11
|
+
# When a failover situation happens (i.e. the primary goes down) the
|
12
|
+
# on_failure callback will be fired. When that callback is fired, the
|
13
|
+
# users of this class are expected to clean up and wait until the
|
14
|
+
# on_recovery deferred is fired. The on_recovery deferred is fired when there
|
15
|
+
# is at least one open connection to a broker available.
|
16
|
+
#
|
17
|
+
# @note this class *will* automatically fail-back when the primary is
|
18
|
+
# re-established.
|
19
|
+
#
|
20
|
+
# TODO: determine how long to wait for connection to primary before failing
|
21
|
+
# over to secondary.
|
22
|
+
#
|
23
|
+
class ConnectionManager
|
24
|
+
include Deferred::Accessors
|
25
|
+
include Logging
|
26
|
+
|
27
|
+
DEFAULT_OPTIONS = {
|
28
|
+
:primary_grace_time => 5.0
|
29
|
+
}
|
30
|
+
|
31
|
+
deferred_event :open, :failure, :close
|
32
|
+
|
33
|
+
attr_reader :primary_opts, :secondary_opts, :primary, :secondary
|
34
|
+
|
35
|
+
attr_accessor :primary_grace_time
|
36
|
+
|
37
|
+
state_machine :state, :initial => :new do
|
38
|
+
|
39
|
+
event :start do
|
40
|
+
transition :new => :start_up
|
41
|
+
end
|
42
|
+
|
43
|
+
# when the secondary connects, it only matters to us if the primary
|
44
|
+
# hasn't been established yet or if it's in some transitional state
|
45
|
+
#
|
46
|
+
event :secondary_connected do
|
47
|
+
transition :start_up => :awaiting_primary, :unless => :primary_connected?
|
48
|
+
transition :degraded => :connected
|
49
|
+
transition :down => :failed_over
|
50
|
+
end
|
51
|
+
|
52
|
+
# fired when we're awaiting the primary to connect and it took too long
|
53
|
+
event :primary_wait_timeout do
|
54
|
+
transition :awaiting_primary => :failed_over
|
55
|
+
end
|
56
|
+
|
57
|
+
# when the primary connects we move into "connected" state and the
|
58
|
+
# clients can now start using the connection
|
59
|
+
#
|
60
|
+
event :primary_connected do
|
61
|
+
transition [:start_up, :down] => :degraded, :unless => :secondary_connected?
|
62
|
+
transition [:awaiting_primary, :failed_over] => :connected
|
63
|
+
end
|
64
|
+
|
65
|
+
event :secondary_failed do
|
66
|
+
# if we're in failed_over mode, transition to down
|
67
|
+
# if we're in connected, just ignore event
|
68
|
+
transition :connected => :degraded
|
69
|
+
transition :failed_over => :down
|
70
|
+
end
|
71
|
+
|
72
|
+
event :primary_failed do
|
73
|
+
transition :connected => :failed_over
|
74
|
+
transition :degraded => :down
|
75
|
+
end
|
76
|
+
|
77
|
+
event :close do
|
78
|
+
transition (all - [:new, :shutting_down, :closed]) => :shutting_down
|
79
|
+
transition :new => :closed
|
80
|
+
end
|
81
|
+
|
82
|
+
event :closed do
|
83
|
+
transition :shutting_down => :closed
|
84
|
+
end
|
85
|
+
|
86
|
+
# start trying to connect to the cluster
|
87
|
+
after_transition :new => :start_up, :do => [:do_connect_primary, :do_connect_secondary]
|
88
|
+
|
89
|
+
# the secondary has connected, start the timer to wait for the primary to connect
|
90
|
+
after_transition :start_up => :awaiting_primary, :do => :start_primary_timer
|
91
|
+
|
92
|
+
# the primary has not connected within the grace period, allow clients to connect
|
93
|
+
after_transition :awaiting_primary => :failed_over, :do => :enter_failed_over_state
|
94
|
+
|
95
|
+
# the primary has connected within the grace period, enter the connected state
|
96
|
+
after_transition :awaiting_primary => :connected, :do => :become_usable!
|
97
|
+
|
98
|
+
#make sure we are up as soon as possible - don't wait for the secondary
|
99
|
+
after_transition :start_up => :degraded, :do => :become_usable!
|
100
|
+
|
101
|
+
#primary has returned
|
102
|
+
after_transition :failed_over => :connected, :do => [:fire_failure_deferred, :become_usable!]
|
103
|
+
|
104
|
+
# the primary fails, secondary is available
|
105
|
+
after_transition :connected => :failed_over, :do => [:fire_failure_deferred, :enter_failed_over_state, :do_connect_primary]
|
106
|
+
|
107
|
+
#the two states we can be in when the secondary is going down
|
108
|
+
after_transition :failed_over => :down, :do => [:fire_failure_deferred, :do_connect_secondary]
|
109
|
+
after_transition :connected => :degraded, :do => :do_connect_secondary
|
110
|
+
|
111
|
+
#primary went down, and secondary was already down
|
112
|
+
after_transition :degraded => :down, :do => [:fire_failure_deferred, :do_connect_primary, :do_connect_secondary]
|
113
|
+
|
114
|
+
#we've been down, now we're coming back up
|
115
|
+
after_transition :down => :failed_over, :do => :enter_failed_over_state
|
116
|
+
after_transition :down => :degraded, :do => :become_usable!
|
117
|
+
|
118
|
+
# shutdown states
|
119
|
+
after_transition [:awaiting_primary, :degraded, :failed_over, :down, :connected] => :shutting_down, :do => :do_close
|
120
|
+
|
121
|
+
# we are really closed
|
122
|
+
after_transition :shutting_down => :closed, :do => :fire_closed_callbacks
|
123
|
+
|
124
|
+
# logging of state transitions
|
125
|
+
around_transition do |cw,block|
|
126
|
+
orig_state = cw.state
|
127
|
+
block.call
|
128
|
+
logger.debug { "CONNECTION MANAGER: moved from #{orig_state} to #{cw.state}" }
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
# @param [AMQP-URL,Hash] primary_cnx valid argument to AMQP.connect to be
|
135
|
+
# used as the primary connection.
|
136
|
+
#
|
137
|
+
# @param [AMQP-URL,Hash] secondary_cnx valid argument to AMQP.connect to be
|
138
|
+
# used as the secondary connection.
|
139
|
+
#
|
140
|
+
# @option opts [Float] :primary_grace_time (5.0) how long we should wait
|
141
|
+
# during startup for the primary to connect before giving up and just
|
142
|
+
# using the secondary connection
|
143
|
+
#
|
144
|
+
def initialize(primary_opts, secondary_opts, opts={})
|
145
|
+
opts = opts.with_indifferent_access.reverse_merge(DEFAULT_OPTIONS)
|
146
|
+
|
147
|
+
@primary_opts = primary_opts
|
148
|
+
@secondary_opts = secondary_opts
|
149
|
+
|
150
|
+
@primary_grace_time = opts[:primary_grace_time]
|
151
|
+
|
152
|
+
if @primary_opts.kind_of?(ConnectionWrapper)
|
153
|
+
@primary = @primary_opts
|
154
|
+
@primary.name ||= 'primary'
|
155
|
+
else
|
156
|
+
@primary = ConnectionWrapper.new(primary_opts, :name => 'primary')
|
157
|
+
end
|
158
|
+
|
159
|
+
if @secondary_opts.kind_of?(ConnectionWrapper)
|
160
|
+
@secondary = @secondary_opts
|
161
|
+
@secondary.name ||= 'secondary'
|
162
|
+
else
|
163
|
+
@secondary = ConnectionWrapper.new(secondary_opts, :name => 'secondary')
|
164
|
+
end
|
165
|
+
|
166
|
+
@active_connection = nil
|
167
|
+
|
168
|
+
super()
|
169
|
+
end
|
170
|
+
|
171
|
+
def start(*a, &b)
|
172
|
+
on_open(&b)
|
173
|
+
super(*a)
|
174
|
+
end
|
175
|
+
|
176
|
+
# connect to the broker, handle reconnections. barf if transition to
|
177
|
+
# :start_up state is not possible
|
178
|
+
def start!(&blk)
|
179
|
+
on_open(&b)
|
180
|
+
super(*a)
|
181
|
+
end
|
182
|
+
|
183
|
+
def close(*a, &b)
|
184
|
+
on_close(&b)
|
185
|
+
super(*a)
|
186
|
+
end
|
187
|
+
|
188
|
+
def close!(*a, &b)
|
189
|
+
on_close(&b)
|
190
|
+
super(*a)
|
191
|
+
end
|
192
|
+
|
193
|
+
def connection
|
194
|
+
@active_connection
|
195
|
+
end
|
196
|
+
|
197
|
+
def primary_connected?
|
198
|
+
primary and primary.connected?
|
199
|
+
end
|
200
|
+
|
201
|
+
def secondary_connected?
|
202
|
+
secondary and secondary.connected?
|
203
|
+
end
|
204
|
+
|
205
|
+
def usable?
|
206
|
+
failed_over? or degraded? or connected?
|
207
|
+
end
|
208
|
+
|
209
|
+
alias :useable? :usable?
|
210
|
+
|
211
|
+
protected
|
212
|
+
def do_connect_primary
|
213
|
+
@primary.on_failure { primary_failed }
|
214
|
+
@primary.on_open { primary_connected }
|
215
|
+
@primary.connect
|
216
|
+
end
|
217
|
+
|
218
|
+
def do_connect_secondary
|
219
|
+
@secondary.on_failure { secondary_failed }
|
220
|
+
@secondary.on_open { secondary_connected }
|
221
|
+
@secondary.connect
|
222
|
+
end
|
223
|
+
|
224
|
+
def do_close
|
225
|
+
logger.debug {"shutting down primary"}
|
226
|
+
@primary.close do
|
227
|
+
logger.debug {"shutting down secondary"}
|
228
|
+
@secondary.close do
|
229
|
+
logger.debug {"secondary shutdown, we're closed'"}
|
230
|
+
closed!
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def start_primary_timer
|
236
|
+
@primary_timer ||= EM::Timer.new(@primary_grace_time) { primary_wait_timeout }
|
237
|
+
end
|
238
|
+
|
239
|
+
def clear_primary_timer!
|
240
|
+
if @primary_timer
|
241
|
+
@primary_timer.cancel
|
242
|
+
@primary_timer = nil
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
def fire_failure_deferred
|
247
|
+
reset_open_event
|
248
|
+
reset_failure_event.succeed
|
249
|
+
end
|
250
|
+
|
251
|
+
def fire_closed_callbacks
|
252
|
+
on_close.succeed
|
253
|
+
end
|
254
|
+
|
255
|
+
def enter_failed_over_state
|
256
|
+
clear_primary_timer!
|
257
|
+
@active_connection = @secondary
|
258
|
+
on_open.succeed
|
259
|
+
end
|
260
|
+
|
261
|
+
def become_usable!
|
262
|
+
clear_primary_timer!
|
263
|
+
@active_connection = @primary
|
264
|
+
on_open.succeed
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
@@ -0,0 +1,210 @@
|
|
1
|
+
module StateMachine
|
2
|
+
class StateMachine::Machine
|
3
|
+
include ::WhoCan::Logging
|
4
|
+
end
|
5
|
+
end
|
6
|
+
|
7
|
+
module WhoCan
|
8
|
+
# encapsulates the retry and failure detection logic. Used by ConnectionManager
|
9
|
+
class ConnectionWrapper
|
10
|
+
include Logging
|
11
|
+
include Deferred::Accessors
|
12
|
+
|
13
|
+
attr_reader :config, :amqp_session, :ekg
|
14
|
+
|
15
|
+
attr_accessor :name
|
16
|
+
|
17
|
+
# how long should we wait between reconnection attempts
|
18
|
+
attr_accessor :reconnect_delay, :shutdown_timeout
|
19
|
+
|
20
|
+
deferred_event :open, :failure, :close, :initial_connection_failure
|
21
|
+
|
22
|
+
state_machine :state, :initial => :new do
|
23
|
+
# moving from no-live-connection to attempting connection
|
24
|
+
after_transition [:new, :retrying] => :connecting, :do => :do_connection
|
25
|
+
|
26
|
+
# callback for handling "failure to connect to the broker"
|
27
|
+
after_transition :connecting => :retrying,
|
28
|
+
:do => :after_initial_connection_failure
|
29
|
+
|
30
|
+
# connection has been established
|
31
|
+
after_transition :connecting => :connected, :do => :after_connected
|
32
|
+
|
33
|
+
# if our connection failed in some way, be sure we shut down ekg
|
34
|
+
before_transition :connected => :retrying, :do => :shutdown_ekg
|
35
|
+
|
36
|
+
# connection fails in some way, hook up retry logic
|
37
|
+
after_transition [:connecting, :connected] => :retrying, :do => [:fire_failure_callbacks, :do_retry]
|
38
|
+
|
39
|
+
# close has been requested
|
40
|
+
after_transition any => :closing, :do => :do_close
|
41
|
+
|
42
|
+
# close has succeeded
|
43
|
+
after_transition [:closing, :new] => :closed, :do => :after_closed
|
44
|
+
|
45
|
+
# logging of state transitions
|
46
|
+
around_transition do |cw,block|
|
47
|
+
orig_state = cw.state
|
48
|
+
block.call
|
49
|
+
# logger.debug { "CONNECTION WRAPPER(#{cw.name}): moved from #{orig_state} to #{cw.state}" }
|
50
|
+
end
|
51
|
+
|
52
|
+
event :connect do
|
53
|
+
transition [:new, :retrying] => :connecting
|
54
|
+
end
|
55
|
+
|
56
|
+
event :connected do
|
57
|
+
transition :connecting => :connected
|
58
|
+
end
|
59
|
+
|
60
|
+
event :initial_connection_failure do
|
61
|
+
transition :connecting => :retrying
|
62
|
+
end
|
63
|
+
|
64
|
+
event :tcp_connection_failed do
|
65
|
+
transition :connected => :retrying
|
66
|
+
end
|
67
|
+
|
68
|
+
event :close do
|
69
|
+
transition [:connecting, :retrying, :connected] => :closing
|
70
|
+
transition :new => :closed
|
71
|
+
end
|
72
|
+
|
73
|
+
event :closed do
|
74
|
+
transition :closing => :closed
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def initialize(config, opts={})
|
80
|
+
@config = config
|
81
|
+
@name = opts[:name]
|
82
|
+
@reconnect_delay = opts[:reconnect_delay] || 5
|
83
|
+
@shutdown_timeout = opts[:shutdown_timeout] || 2.0
|
84
|
+
|
85
|
+
@ekg = nil
|
86
|
+
@retry_count = 0
|
87
|
+
@amqp_session = nil
|
88
|
+
@on_initial_connection_failure_cbs = []
|
89
|
+
super()
|
90
|
+
end
|
91
|
+
|
92
|
+
# XXX: figure out how to do this in a cleaner fashion
|
93
|
+
def connect(*a, &b)
|
94
|
+
on_open(&b)
|
95
|
+
super(*a)
|
96
|
+
end
|
97
|
+
|
98
|
+
def connect!(*a, &b)
|
99
|
+
on_open(&b)
|
100
|
+
super(*a)
|
101
|
+
end
|
102
|
+
|
103
|
+
def close(*a, &b)
|
104
|
+
on_close(&b) # hook deferred into this event
|
105
|
+
super(*a)
|
106
|
+
end
|
107
|
+
|
108
|
+
def close!(*a, &b)
|
109
|
+
on_close(&b) # hook deferred into this event
|
110
|
+
super(*a)
|
111
|
+
end
|
112
|
+
|
113
|
+
protected
|
114
|
+
def do_connection
|
115
|
+
logger.info { "#{name} attempting connection to #{@config.inspect}" }
|
116
|
+
|
117
|
+
AMQP.connect(@config, :on_tcp_connection_failure => method(:initial_connection_failure!).to_proc) do |sess|
|
118
|
+
@ekg = Heartbeater::EKG.new(sess)
|
119
|
+
|
120
|
+
@ekg.on_heartbeat_failure { tcp_connection_failed }
|
121
|
+
|
122
|
+
@ekg.start!
|
123
|
+
@amqp_session = sess
|
124
|
+
|
125
|
+
connected
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def do_close
|
130
|
+
@retry_timer.cancel if @retry_timer
|
131
|
+
|
132
|
+
logger.debug { "wrapper(#{name}): do_close called" }
|
133
|
+
|
134
|
+
if @amqp_session || @ekg
|
135
|
+
logger.debug { "wrapper(#{name}): closing ekg!" }
|
136
|
+
shutdown_ekg { closed }
|
137
|
+
else
|
138
|
+
closed
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def fire_failure_callbacks
|
143
|
+
return if closing? or closed?
|
144
|
+
logger.debug { "wrapper(#{name}): fire_failure_callbacks" }
|
145
|
+
reset_open_event
|
146
|
+
reset_failure_event.succeed
|
147
|
+
end
|
148
|
+
|
149
|
+
def shutdown_ekg(&blk)
|
150
|
+
Deferred::Default.new.tap do |my_dfr|
|
151
|
+
if @ekg
|
152
|
+
amqp_session, @amqp_session = @amqp_session, nil
|
153
|
+
|
154
|
+
my_dfr.timeout(@shutdown_timeout) # we need to ensure that we will time out
|
155
|
+
my_dfr.ensure_that(&blk) if blk
|
156
|
+
|
157
|
+
if amqp_session
|
158
|
+
@ekg.on_shutdown.ensure_that do
|
159
|
+
logger.debug { "wrapper(#{name}): ekg shutdown completed" }
|
160
|
+
|
161
|
+
amqp_session.on_closed { my_dfr.succeed }
|
162
|
+
|
163
|
+
amqp_session.close
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
logger.debug { "wrapper(#{name}): shutting down ekg" }
|
168
|
+
|
169
|
+
@ekg.on_shutdown.errback do |e|
|
170
|
+
logger.debug { "wrapper(#{name}): ekg shutdown timed out!" }
|
171
|
+
end
|
172
|
+
|
173
|
+
@ekg.on_shutdown.timeout(@shutdown_timeout, TimeoutError) # ensure that ekg shutdown won't hang
|
174
|
+
@ekg.shutdown!
|
175
|
+
@ekg = nil
|
176
|
+
else
|
177
|
+
my_dfr.succeed
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def after_connected
|
183
|
+
on_open.succeed
|
184
|
+
end
|
185
|
+
|
186
|
+
def after_closed
|
187
|
+
on_close.succeed
|
188
|
+
end
|
189
|
+
|
190
|
+
def after_initial_connection_failure
|
191
|
+
logger.warn { "wrapper(#{name}): connection to #{@config.inspect} failed, this is either a misconfiguration or the server is down" }
|
192
|
+
logger.warn { "wrapper(#{name}): retrying after a delay of #{reconnect_delay}s" }
|
193
|
+
|
194
|
+
reset_initial_connection_failure_event.succeed
|
195
|
+
end
|
196
|
+
|
197
|
+
def do_retry
|
198
|
+
return if closing? or closed?
|
199
|
+
|
200
|
+
logger.debug { "wrapper(#{name}): do_retry setting up reconnection timer" }
|
201
|
+
|
202
|
+
@retry_timer = EM::Timer.new(reconnect_delay) do
|
203
|
+
@retry_count += 1
|
204
|
+
logger.debug { "wrapper(#{name}): retry attempt: #{@retry_count}" }
|
205
|
+
connect!
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|