beetle 1.0.3 → 2.0.0rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +15 -7
- data/REDIS_AUTO_FAILOVER.rdoc +8 -16
- data/RELEASE_NOTES.rdoc +9 -0
- data/beetle.gemspec +22 -10
- data/features/support/system_notification_logger +29 -12
- data/features/support/test_daemons/redis.rb +1 -1
- data/features/support/test_daemons/redis_configuration_client.rb +3 -3
- data/features/support/test_daemons/redis_configuration_server.rb +2 -2
- data/lib/beetle/deduplication_store.rb +0 -66
- data/lib/beetle/message.rb +1 -1
- data/lib/beetle/publisher.rb +2 -1
- data/lib/beetle/redis_master_file.rb +0 -5
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +0 -48
- data/test/beetle/message_test.rb +1 -31
- data/test/beetle/publisher_test.rb +2 -1
- metadata +165 -26
- data/bin/beetle +0 -9
- data/lib/beetle/commands.rb +0 -35
- data/lib/beetle/commands/configuration_client.rb +0 -98
- data/lib/beetle/commands/configuration_server.rb +0 -98
- data/lib/beetle/commands/garbage_collect_deduplication_store.rb +0 -52
- data/lib/beetle/redis_configuration_client.rb +0 -157
- data/lib/beetle/redis_configuration_http_server.rb +0 -152
- data/lib/beetle/redis_configuration_server.rb +0 -438
- data/lib/beetle/redis_server_info.rb +0 -66
- data/script/docker-run-beetle-tests- +0 -5
- data/test/beetle/redis_configuration_client_test.rb +0 -118
- data/test/beetle/redis_configuration_server_test.rb +0 -381
@@ -1,438 +0,0 @@
|
|
1
|
-
module Beetle
|
2
|
-
# A RedisConfigurationServer is the supervisor part of beetle's
|
3
|
-
# redis failover solution.
|
4
|
-
#
|
5
|
-
# A single instance of RedisConfigurationServer works as a supervisor for
|
6
|
-
# several RedisConfigurationClient instances. It is responsible for watching
|
7
|
-
# the redis master and electing and publishing a new master in case of failure.
|
8
|
-
#
|
9
|
-
# It will make sure that all configured RedisConfigurationClient instances
|
10
|
-
# do not use the old master anymore before making a switch, to prevent
|
11
|
-
# inconsistent data.
|
12
|
-
#
|
13
|
-
# Usually started via <tt>beetle configuration_server</tt> command.
|
14
|
-
class RedisConfigurationServer
|
15
|
-
include Logging
|
16
|
-
include RedisMasterFile
|
17
|
-
|
18
|
-
# the current redis master
|
19
|
-
attr_reader :current_master
|
20
|
-
|
21
|
-
# the current token used to detect correct message order
|
22
|
-
attr_reader :current_token
|
23
|
-
|
24
|
-
# the list of known client ids
|
25
|
-
attr_reader :client_ids
|
26
|
-
|
27
|
-
# the list of client ids we have received but we don't know about
|
28
|
-
attr_reader :unknown_client_ids
|
29
|
-
|
30
|
-
# clients last seen time stamps
|
31
|
-
attr_reader :clients_last_seen
|
32
|
-
|
33
|
-
# clients are presumed dead after this many seconds
|
34
|
-
attr_accessor :client_dead_threshold
|
35
|
-
|
36
|
-
def initialize #:nodoc:
|
37
|
-
@client_ids = Set.new(config.redis_configuration_client_ids.split(","))
|
38
|
-
@unknown_client_ids = Set.new
|
39
|
-
@clients_last_seen = {}
|
40
|
-
@current_token = (Time.now.to_f * 1000).to_i
|
41
|
-
@client_pong_ids_received = Set.new
|
42
|
-
@client_invalidated_ids_received = Set.new
|
43
|
-
@client_dead_threshold = config.redis_failover_client_dead_interval
|
44
|
-
MessageDispatcher.configuration_server = self
|
45
|
-
end
|
46
|
-
|
47
|
-
# Redis system status information (an instance of class RedisServerInfo)
|
48
|
-
def redis
|
49
|
-
@redis ||= RedisServerInfo.new(config, :timeout => 3)
|
50
|
-
end
|
51
|
-
|
52
|
-
# Beetle::Client instance for communication with the RedisConfigurationServer
|
53
|
-
def beetle
|
54
|
-
@beetle ||= build_beetle
|
55
|
-
end
|
56
|
-
|
57
|
-
def config #:nodoc:
|
58
|
-
beetle.config
|
59
|
-
end
|
60
|
-
|
61
|
-
# returns a hash describing the current server status
|
62
|
-
def status
|
63
|
-
{
|
64
|
-
:beetle_version => Beetle::VERSION,
|
65
|
-
:configured_brokers => config.servers.split(/\s*,\s*/),
|
66
|
-
:configured_client_ids => client_ids.to_a.sort,
|
67
|
-
:configured_redis_servers => config.redis_servers.split(/\s*,\s*/),
|
68
|
-
:redis_master => current_master.try(:server).to_s,
|
69
|
-
:redis_master_available? => master_available?,
|
70
|
-
:redis_slaves_available => available_slaves.map(&:server),
|
71
|
-
:switch_in_progress => paused?,
|
72
|
-
:unknown_client_ids => unknown_client_ids.to_a,
|
73
|
-
:unresponsive_clients => unresponsive_clients.map{|c,t| "#{c}:#{t.to_i}"},
|
74
|
-
:unseen_client_ids => unseen_client_ids
|
75
|
-
}
|
76
|
-
end
|
77
|
-
|
78
|
-
# return array of client ids which never sent a ping
|
79
|
-
def unseen_client_ids
|
80
|
-
(client_ids - clients_last_seen.keys.to_set).to_a.sort
|
81
|
-
end
|
82
|
-
|
83
|
-
# returns an array of [client_id, silent time in seconds] which haven't sent
|
84
|
-
# a message for more than <tt>client_dead_threshold</tt> seconds, sorted by
|
85
|
-
# silent time in descending order.
|
86
|
-
def unresponsive_clients
|
87
|
-
now = Time.now
|
88
|
-
threshold = now - client_dead_threshold
|
89
|
-
clients_last_seen.select{|_,t| t < threshold}.map{|c,t| [c, now - t]}.sort_by{|_,t| t}.reverse
|
90
|
-
end
|
91
|
-
|
92
|
-
# start watching redis
|
93
|
-
def start
|
94
|
-
verify_redis_master_file_string
|
95
|
-
check_redis_configuration
|
96
|
-
redis.refresh
|
97
|
-
determine_initial_master
|
98
|
-
log_start
|
99
|
-
beetle.listen do
|
100
|
-
master_watcher.watch
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
# test if redis is currently being watched
|
105
|
-
def paused?
|
106
|
-
master_watcher.paused?
|
107
|
-
end
|
108
|
-
|
109
|
-
# called by the message dispatcher when a "pong" message from a RedisConfigurationClient is received
|
110
|
-
def pong(payload)
|
111
|
-
id = payload["id"]
|
112
|
-
client_seen(id)
|
113
|
-
token = payload["token"]
|
114
|
-
return unless validate_pong_client_id(id)
|
115
|
-
logger.info "Received pong message from id '#{id}' with token '#{token}'"
|
116
|
-
return unless redeem_token(token)
|
117
|
-
@client_pong_ids_received << id
|
118
|
-
if all_client_pong_ids_received?
|
119
|
-
logger.info "All client pong messages received"
|
120
|
-
@available_timer.cancel if @available_timer
|
121
|
-
invalidate_current_master
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
# called by the message dispatcher when a "client_started" message from a RedisConfigurationClient is received
|
126
|
-
def client_started(payload)
|
127
|
-
id = payload["id"]
|
128
|
-
client_seen(id)
|
129
|
-
if client_id_valid?(id)
|
130
|
-
logger.info "Received client_started message from id '#{id}'"
|
131
|
-
else
|
132
|
-
add_unknown_client_id(id)
|
133
|
-
msg = "Received client_started message from unknown id '#{id}'"
|
134
|
-
logger.error msg
|
135
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
|
-
# called by the message dispatcher when a "heartbeat" message from a RedisConfigurationClient is received
|
140
|
-
def heartbeat(payload)
|
141
|
-
id = payload["id"]
|
142
|
-
client_seen(id)
|
143
|
-
if client_id_valid?(id)
|
144
|
-
logger.info "Received heartbeat message from id '#{id}'"
|
145
|
-
else
|
146
|
-
add_unknown_client_id(id)
|
147
|
-
msg = "Received heartbeat message from unknown id '#{id}'"
|
148
|
-
logger.error msg
|
149
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
150
|
-
end
|
151
|
-
end
|
152
|
-
|
153
|
-
# called by the message dispatcher when a "client_invalidated" message from a RedisConfigurationClient is received
|
154
|
-
def client_invalidated(payload)
|
155
|
-
id = payload["id"]
|
156
|
-
client_seen(id)
|
157
|
-
token = payload["token"]
|
158
|
-
add_unknown_client_id(id) unless client_id_valid?(id)
|
159
|
-
logger.info "Received client_invalidated message from id '#{id}' with token '#{token}'"
|
160
|
-
return unless redeem_token(token)
|
161
|
-
@client_invalidated_ids_received << id
|
162
|
-
if all_client_invalidated_ids_received?
|
163
|
-
logger.debug "All client invalidated messages received"
|
164
|
-
@invalidate_timer.cancel if @invalidate_timer
|
165
|
-
switch_master
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
# called from RedisWatcher when watched redis becomes unavailable
|
170
|
-
def master_unavailable!
|
171
|
-
msg = "Redis master '#{current_master.server}' not available"
|
172
|
-
master_watcher.pause
|
173
|
-
logger.warn(msg)
|
174
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
175
|
-
|
176
|
-
if @client_ids.empty?
|
177
|
-
switch_master
|
178
|
-
else
|
179
|
-
start_invalidation
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
# called from RedisWatcher when watched redis is available
|
184
|
-
def master_available!
|
185
|
-
publish_master(current_master)
|
186
|
-
configure_slaves(current_master)
|
187
|
-
end
|
188
|
-
|
189
|
-
# check whether the current master is still online and still a master
|
190
|
-
def master_available?
|
191
|
-
redis.masters.include?(current_master)
|
192
|
-
end
|
193
|
-
|
194
|
-
# list of available redis slaves
|
195
|
-
def available_slaves
|
196
|
-
redis.slaves
|
197
|
-
end
|
198
|
-
|
199
|
-
# initiate a master switch if the current master is not available and no switch is in progress
|
200
|
-
def initiate_master_switch
|
201
|
-
redis.refresh
|
202
|
-
available, switch_in_progress = master_available?, paused?
|
203
|
-
logger.debug "Initiating master switch: already in progress = #{switch_in_progress}"
|
204
|
-
unless available || switch_in_progress
|
205
|
-
master_unavailable!
|
206
|
-
end
|
207
|
-
!available || switch_in_progress
|
208
|
-
end
|
209
|
-
|
210
|
-
private
|
211
|
-
|
212
|
-
# prevent memory overflows caused by evil or buggy clients
|
213
|
-
MAX_UNKNOWN_CLIENT_IDS = 20
|
214
|
-
|
215
|
-
def add_unknown_client_id(id)
|
216
|
-
ids = @unknown_client_ids
|
217
|
-
while ids.size >= MAX_UNKNOWN_CLIENT_IDS
|
218
|
-
old_id = ids.first
|
219
|
-
ids.delete(old_id)
|
220
|
-
clients_last_seen.delete(old_id)
|
221
|
-
end
|
222
|
-
ids << id
|
223
|
-
end
|
224
|
-
|
225
|
-
def client_seen(client_id)
|
226
|
-
clients_last_seen[client_id] = Time.now
|
227
|
-
end
|
228
|
-
|
229
|
-
def check_redis_configuration
|
230
|
-
raise ConfigurationError.new("Redis failover needs two or more redis servers") if redis.instances.size < 2
|
231
|
-
end
|
232
|
-
|
233
|
-
def log_start
|
234
|
-
logger.info "RedisConfigurationServer starting"
|
235
|
-
logger.info "AMQP servers : #{config.servers}"
|
236
|
-
logger.info "Client ids : #{config.redis_configuration_client_ids}"
|
237
|
-
logger.info "Redis servers : #{config.redis_servers}"
|
238
|
-
logger.info "Redis master : #{current_master.server}"
|
239
|
-
end
|
240
|
-
|
241
|
-
def build_beetle
|
242
|
-
system = Beetle.config.system_name
|
243
|
-
Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
|
244
|
-
# messages sent
|
245
|
-
config.message :ping
|
246
|
-
config.message :invalidate
|
247
|
-
config.message :reconfigure
|
248
|
-
config.message :system_notification
|
249
|
-
# messages received
|
250
|
-
config.message :client_started
|
251
|
-
config.message :pong
|
252
|
-
config.message :client_invalidated
|
253
|
-
config.message :heartbeat
|
254
|
-
# queue setup
|
255
|
-
config.queue :server, :key => 'pong', :amqp_name => "#{system}_configuration_server"
|
256
|
-
config.binding :server, :key => 'client_started'
|
257
|
-
config.binding :server, :key => 'client_invalidated'
|
258
|
-
config.binding :server, :key => 'heartbeat'
|
259
|
-
config.handler :server, MessageDispatcher
|
260
|
-
end
|
261
|
-
end
|
262
|
-
|
263
|
-
def master_watcher
|
264
|
-
@master_watcher ||= RedisWatcher.new(self)
|
265
|
-
end
|
266
|
-
|
267
|
-
def determine_initial_master
|
268
|
-
if master_file_exists? && @current_master = redis_master_from_master_file
|
269
|
-
if redis.slaves.include?(current_master)
|
270
|
-
master_unavailable!
|
271
|
-
elsif redis.unknowns.include?(current_master)
|
272
|
-
master_unavailable!
|
273
|
-
end
|
274
|
-
else
|
275
|
-
write_redis_master_file(current_master.server) if @current_master = redis.auto_detect_master
|
276
|
-
end
|
277
|
-
current_master or raise NoRedisMaster.new("Failed to determine initial redis master")
|
278
|
-
end
|
279
|
-
|
280
|
-
def determine_new_master
|
281
|
-
redis.unknowns.include?(current_master) ? redis.slaves_of(current_master).first : current_master
|
282
|
-
end
|
283
|
-
|
284
|
-
def validate_pong_client_id(client_id)
|
285
|
-
unless known_client = client_id_valid?(client_id)
|
286
|
-
add_unknown_client_id(client_id)
|
287
|
-
msg = "Received pong message from unknown id '#{client_id}'"
|
288
|
-
logger.error msg
|
289
|
-
logger.info "Sending system_notification message with text: #{msg}"
|
290
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
291
|
-
end
|
292
|
-
known_client
|
293
|
-
end
|
294
|
-
|
295
|
-
def client_id_valid?(client_id)
|
296
|
-
@client_ids.include?(client_id)
|
297
|
-
end
|
298
|
-
|
299
|
-
def redeem_token(token)
|
300
|
-
valid_token = token == @current_token
|
301
|
-
logger.info "Ignored message (token was '#{token.inspect}', but expected '#{@current_token.inspect}')" unless valid_token
|
302
|
-
valid_token
|
303
|
-
end
|
304
|
-
|
305
|
-
def start_invalidation
|
306
|
-
@client_pong_ids_received.clear
|
307
|
-
@client_invalidated_ids_received.clear
|
308
|
-
check_all_clients_available
|
309
|
-
end
|
310
|
-
|
311
|
-
def check_all_clients_available
|
312
|
-
generate_new_token
|
313
|
-
logger.info "Sending ping message with token '#{@current_token}'"
|
314
|
-
beetle.publish(:ping, payload_with_current_token)
|
315
|
-
@available_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
|
316
|
-
end
|
317
|
-
|
318
|
-
def invalidate_current_master
|
319
|
-
generate_new_token
|
320
|
-
logger.info "Sending invalidate message with token '#{@current_token}'"
|
321
|
-
beetle.publish(:invalidate, payload_with_current_token)
|
322
|
-
@invalidate_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
|
323
|
-
end
|
324
|
-
|
325
|
-
def cancel_invalidation
|
326
|
-
logger.warn "Redis master invalidation cancelled: 'pong' received from '#{@client_pong_ids_received.to_a.join(',')}', 'client_invalidated' received from '#{@client_invalidated_ids_received.to_a.join(',')}'"
|
327
|
-
generate_new_token
|
328
|
-
master_watcher.continue
|
329
|
-
end
|
330
|
-
|
331
|
-
def generate_new_token
|
332
|
-
@current_token += 1
|
333
|
-
end
|
334
|
-
|
335
|
-
def payload_with_current_token(message = {})
|
336
|
-
message["token"] = @current_token
|
337
|
-
message.to_json
|
338
|
-
end
|
339
|
-
|
340
|
-
def all_client_pong_ids_received?
|
341
|
-
@client_ids == @client_pong_ids_received
|
342
|
-
end
|
343
|
-
|
344
|
-
def all_client_invalidated_ids_received?
|
345
|
-
@client_ids == @client_invalidated_ids_received
|
346
|
-
end
|
347
|
-
|
348
|
-
def switch_master
|
349
|
-
if new_master = determine_new_master
|
350
|
-
msg = "Setting redis master to '#{new_master.server}' (was '#{current_master.server}')"
|
351
|
-
logger.warn msg
|
352
|
-
logger.info "Sending system_notification message with text: #{msg}"
|
353
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
354
|
-
|
355
|
-
new_master.master!
|
356
|
-
write_redis_master_file(new_master.server)
|
357
|
-
@current_master = new_master
|
358
|
-
else
|
359
|
-
msg = "Redis master could not be switched, no slave available to become new master, promoting old master"
|
360
|
-
logger.error msg
|
361
|
-
logger.info "Sending system_notification message with text: #{msg}"
|
362
|
-
beetle.publish(:system_notification, {"message" => msg}.to_json)
|
363
|
-
end
|
364
|
-
|
365
|
-
publish_master(current_master)
|
366
|
-
master_watcher.continue
|
367
|
-
end
|
368
|
-
|
369
|
-
def publish_master(master)
|
370
|
-
logger.info "Sending reconfigure message with server '#{master.server}'"
|
371
|
-
beetle.publish(:reconfigure, payload_with_current_token({"server" => master.server}))
|
372
|
-
end
|
373
|
-
|
374
|
-
def configure_slaves(master)
|
375
|
-
(redis.masters-[master]).each do |r|
|
376
|
-
logger.info "Reconfiguring '#{r.server}' as a slave of '#{master.server}'"
|
377
|
-
r.slave_of!(master.host, master.port)
|
378
|
-
end
|
379
|
-
end
|
380
|
-
|
381
|
-
# Periodically checks a redis server for availability
|
382
|
-
class RedisWatcher #:nodoc:
|
383
|
-
include Logging
|
384
|
-
|
385
|
-
def initialize(configuration_server)
|
386
|
-
@configuration_server = configuration_server
|
387
|
-
@retries = 0
|
388
|
-
@paused = true
|
389
|
-
@master_retry_interval = configuration_server.config.redis_configuration_master_retry_interval
|
390
|
-
@master_retries = configuration_server.config.redis_configuration_master_retries
|
391
|
-
end
|
392
|
-
|
393
|
-
def pause
|
394
|
-
logger.info "Pause checking availability of redis servers"
|
395
|
-
@watch_timer.cancel if @watch_timer
|
396
|
-
@watch_timer = nil
|
397
|
-
@paused = true
|
398
|
-
end
|
399
|
-
|
400
|
-
def watch
|
401
|
-
@watch_timer ||=
|
402
|
-
begin
|
403
|
-
logger.info "Start watching redis servers every #{@master_retry_interval} seconds"
|
404
|
-
EventMachine::add_periodic_timer(@master_retry_interval) { check_availability }
|
405
|
-
end
|
406
|
-
@paused = false
|
407
|
-
end
|
408
|
-
alias continue watch
|
409
|
-
|
410
|
-
def paused?
|
411
|
-
@paused
|
412
|
-
end
|
413
|
-
|
414
|
-
private
|
415
|
-
|
416
|
-
def check_availability
|
417
|
-
@configuration_server.redis.refresh
|
418
|
-
if @configuration_server.master_available?
|
419
|
-
@configuration_server.master_available!
|
420
|
-
else
|
421
|
-
logger.warn "Redis master not available! (Retries left: #{@master_retries - (@retries + 1)})"
|
422
|
-
if (@retries+=1) >= @master_retries
|
423
|
-
@retries = 0
|
424
|
-
@configuration_server.master_unavailable!
|
425
|
-
end
|
426
|
-
end
|
427
|
-
end
|
428
|
-
end
|
429
|
-
|
430
|
-
# Dispatches messages from the queue to methods in RedisConfigurationServer
|
431
|
-
class MessageDispatcher < Beetle::Handler #:nodoc:
|
432
|
-
cattr_accessor :configuration_server
|
433
|
-
def process
|
434
|
-
@@configuration_server.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
|
435
|
-
end
|
436
|
-
end
|
437
|
-
end
|
438
|
-
end
|
@@ -1,66 +0,0 @@
|
|
1
|
-
module Beetle
|
2
|
-
# class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
|
3
|
-
class RedisServerInfo
|
4
|
-
include Logging
|
5
|
-
|
6
|
-
def initialize(config, options) #:nodoc:
|
7
|
-
@config = config
|
8
|
-
@options = options
|
9
|
-
reset
|
10
|
-
end
|
11
|
-
|
12
|
-
# all configured redis servers
|
13
|
-
def instances
|
14
|
-
@instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
|
15
|
-
end
|
16
|
-
|
17
|
-
# fetches the server from the instances whith the given <tt>server</tt> string
|
18
|
-
def find(server)
|
19
|
-
instances.find{|r| r.server == server}
|
20
|
-
end
|
21
|
-
|
22
|
-
# refresh connectivity/role information
|
23
|
-
def refresh
|
24
|
-
logger.debug "Updating redis server info"
|
25
|
-
reset
|
26
|
-
instances.each {|r| @server_info[r.role] << r}
|
27
|
-
end
|
28
|
-
|
29
|
-
# subset of instances which are masters
|
30
|
-
def masters
|
31
|
-
@server_info["master"]
|
32
|
-
end
|
33
|
-
|
34
|
-
# subset of instances which are slaves
|
35
|
-
def slaves
|
36
|
-
@server_info["slave"]
|
37
|
-
end
|
38
|
-
|
39
|
-
# subset of instances which are not reachable
|
40
|
-
def unknowns
|
41
|
-
@server_info["unknown"]
|
42
|
-
end
|
43
|
-
|
44
|
-
# subset of instances which are set up as slaves of the given <tt>master</tt>
|
45
|
-
def slaves_of(master)
|
46
|
-
slaves.select{|r| r.slave_of?(master.host, master.port)}
|
47
|
-
end
|
48
|
-
|
49
|
-
# determine a master if we have one master and all other servers are slaves
|
50
|
-
def auto_detect_master
|
51
|
-
return nil unless master_and_slaves_reachable?
|
52
|
-
masters.first
|
53
|
-
end
|
54
|
-
|
55
|
-
# check whether all redis servers are up and configured
|
56
|
-
def master_and_slaves_reachable?
|
57
|
-
masters.size == 1 && slaves.size == instances.size - 1
|
58
|
-
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
def reset
|
63
|
-
@server_info = Hash.new {|h,k| h[k]= []}
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|