beetle 1.0.3 → 2.0.0rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,438 +0,0 @@
1
- module Beetle
2
- # A RedisConfigurationServer is the supervisor part of beetle's
3
- # redis failover solution.
4
- #
5
- # A single instance of RedisConfigurationServer works as a supervisor for
6
- # several RedisConfigurationClient instances. It is responsible for watching
7
- # the redis master and electing and publishing a new master in case of failure.
8
- #
9
- # It will make sure that all configured RedisConfigurationClient instances
10
- # do not use the old master anymore before making a switch, to prevent
11
- # inconsistent data.
12
- #
13
- # Usually started via <tt>beetle configuration_server</tt> command.
14
- class RedisConfigurationServer
15
- include Logging
16
- include RedisMasterFile
17
-
18
- # the current redis master
19
- attr_reader :current_master
20
-
21
- # the current token used to detect correct message order
22
- attr_reader :current_token
23
-
24
- # the list of known client ids
25
- attr_reader :client_ids
26
-
27
- # the list of client ids we have received but we don't know about
28
- attr_reader :unknown_client_ids
29
-
30
- # clients last seen time stamps
31
- attr_reader :clients_last_seen
32
-
33
- # clients are presumed dead after this many seconds
34
- attr_accessor :client_dead_threshold
35
-
36
- def initialize #:nodoc:
37
- @client_ids = Set.new(config.redis_configuration_client_ids.split(","))
38
- @unknown_client_ids = Set.new
39
- @clients_last_seen = {}
40
- @current_token = (Time.now.to_f * 1000).to_i
41
- @client_pong_ids_received = Set.new
42
- @client_invalidated_ids_received = Set.new
43
- @client_dead_threshold = config.redis_failover_client_dead_interval
44
- MessageDispatcher.configuration_server = self
45
- end
46
-
47
- # Redis system status information (an instance of class RedisServerInfo)
48
- def redis
49
- @redis ||= RedisServerInfo.new(config, :timeout => 3)
50
- end
51
-
52
- # Beetle::Client instance for communication with the RedisConfigurationServer
53
- def beetle
54
- @beetle ||= build_beetle
55
- end
56
-
57
- def config #:nodoc:
58
- beetle.config
59
- end
60
-
61
- # returns a hash describing the current server status
62
- def status
63
- {
64
- :beetle_version => Beetle::VERSION,
65
- :configured_brokers => config.servers.split(/\s*,\s*/),
66
- :configured_client_ids => client_ids.to_a.sort,
67
- :configured_redis_servers => config.redis_servers.split(/\s*,\s*/),
68
- :redis_master => current_master.try(:server).to_s,
69
- :redis_master_available? => master_available?,
70
- :redis_slaves_available => available_slaves.map(&:server),
71
- :switch_in_progress => paused?,
72
- :unknown_client_ids => unknown_client_ids.to_a,
73
- :unresponsive_clients => unresponsive_clients.map{|c,t| "#{c}:#{t.to_i}"},
74
- :unseen_client_ids => unseen_client_ids
75
- }
76
- end
77
-
78
- # return array of client ids which never sent a ping
79
- def unseen_client_ids
80
- (client_ids - clients_last_seen.keys.to_set).to_a.sort
81
- end
82
-
83
- # returns an array of [client_id, silent time in seconds] which haven't sent
84
- # a message for more than <tt>client_dead_threshold</tt> seconds, sorted by
85
- # silent time in descending order.
86
- def unresponsive_clients
87
- now = Time.now
88
- threshold = now - client_dead_threshold
89
- clients_last_seen.select{|_,t| t < threshold}.map{|c,t| [c, now - t]}.sort_by{|_,t| t}.reverse
90
- end
91
-
92
- # start watching redis
93
- def start
94
- verify_redis_master_file_string
95
- check_redis_configuration
96
- redis.refresh
97
- determine_initial_master
98
- log_start
99
- beetle.listen do
100
- master_watcher.watch
101
- end
102
- end
103
-
104
- # test if redis is currently being watched
105
- def paused?
106
- master_watcher.paused?
107
- end
108
-
109
- # called by the message dispatcher when a "pong" message from a RedisConfigurationClient is received
110
- def pong(payload)
111
- id = payload["id"]
112
- client_seen(id)
113
- token = payload["token"]
114
- return unless validate_pong_client_id(id)
115
- logger.info "Received pong message from id '#{id}' with token '#{token}'"
116
- return unless redeem_token(token)
117
- @client_pong_ids_received << id
118
- if all_client_pong_ids_received?
119
- logger.info "All client pong messages received"
120
- @available_timer.cancel if @available_timer
121
- invalidate_current_master
122
- end
123
- end
124
-
125
- # called by the message dispatcher when a "client_started" message from a RedisConfigurationClient is received
126
- def client_started(payload)
127
- id = payload["id"]
128
- client_seen(id)
129
- if client_id_valid?(id)
130
- logger.info "Received client_started message from id '#{id}'"
131
- else
132
- add_unknown_client_id(id)
133
- msg = "Received client_started message from unknown id '#{id}'"
134
- logger.error msg
135
- beetle.publish(:system_notification, {"message" => msg}.to_json)
136
- end
137
- end
138
-
139
- # called by the message dispatcher when a "heartbeat" message from a RedisConfigurationClient is received
140
- def heartbeat(payload)
141
- id = payload["id"]
142
- client_seen(id)
143
- if client_id_valid?(id)
144
- logger.info "Received heartbeat message from id '#{id}'"
145
- else
146
- add_unknown_client_id(id)
147
- msg = "Received heartbeat message from unknown id '#{id}'"
148
- logger.error msg
149
- beetle.publish(:system_notification, {"message" => msg}.to_json)
150
- end
151
- end
152
-
153
- # called by the message dispatcher when a "client_invalidated" message from a RedisConfigurationClient is received
154
- def client_invalidated(payload)
155
- id = payload["id"]
156
- client_seen(id)
157
- token = payload["token"]
158
- add_unknown_client_id(id) unless client_id_valid?(id)
159
- logger.info "Received client_invalidated message from id '#{id}' with token '#{token}'"
160
- return unless redeem_token(token)
161
- @client_invalidated_ids_received << id
162
- if all_client_invalidated_ids_received?
163
- logger.debug "All client invalidated messages received"
164
- @invalidate_timer.cancel if @invalidate_timer
165
- switch_master
166
- end
167
- end
168
-
169
- # called from RedisWatcher when watched redis becomes unavailable
170
- def master_unavailable!
171
- msg = "Redis master '#{current_master.server}' not available"
172
- master_watcher.pause
173
- logger.warn(msg)
174
- beetle.publish(:system_notification, {"message" => msg}.to_json)
175
-
176
- if @client_ids.empty?
177
- switch_master
178
- else
179
- start_invalidation
180
- end
181
- end
182
-
183
- # called from RedisWatcher when watched redis is available
184
- def master_available!
185
- publish_master(current_master)
186
- configure_slaves(current_master)
187
- end
188
-
189
- # check whether the current master is still online and still a master
190
- def master_available?
191
- redis.masters.include?(current_master)
192
- end
193
-
194
- # list of available redis slaves
195
- def available_slaves
196
- redis.slaves
197
- end
198
-
199
- # initiate a master switch if the current master is not available and no switch is in progress
200
- def initiate_master_switch
201
- redis.refresh
202
- available, switch_in_progress = master_available?, paused?
203
- logger.debug "Initiating master switch: already in progress = #{switch_in_progress}"
204
- unless available || switch_in_progress
205
- master_unavailable!
206
- end
207
- !available || switch_in_progress
208
- end
209
-
210
- private
211
-
212
- # prevent memory overflows caused by evil or buggy clients
213
- MAX_UNKNOWN_CLIENT_IDS = 20
214
-
215
- def add_unknown_client_id(id)
216
- ids = @unknown_client_ids
217
- while ids.size >= MAX_UNKNOWN_CLIENT_IDS
218
- old_id = ids.first
219
- ids.delete(old_id)
220
- clients_last_seen.delete(old_id)
221
- end
222
- ids << id
223
- end
224
-
225
- def client_seen(client_id)
226
- clients_last_seen[client_id] = Time.now
227
- end
228
-
229
- def check_redis_configuration
230
- raise ConfigurationError.new("Redis failover needs two or more redis servers") if redis.instances.size < 2
231
- end
232
-
233
- def log_start
234
- logger.info "RedisConfigurationServer starting"
235
- logger.info "AMQP servers : #{config.servers}"
236
- logger.info "Client ids : #{config.redis_configuration_client_ids}"
237
- logger.info "Redis servers : #{config.redis_servers}"
238
- logger.info "Redis master : #{current_master.server}"
239
- end
240
-
241
- def build_beetle
242
- system = Beetle.config.system_name
243
- Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
244
- # messages sent
245
- config.message :ping
246
- config.message :invalidate
247
- config.message :reconfigure
248
- config.message :system_notification
249
- # messages received
250
- config.message :client_started
251
- config.message :pong
252
- config.message :client_invalidated
253
- config.message :heartbeat
254
- # queue setup
255
- config.queue :server, :key => 'pong', :amqp_name => "#{system}_configuration_server"
256
- config.binding :server, :key => 'client_started'
257
- config.binding :server, :key => 'client_invalidated'
258
- config.binding :server, :key => 'heartbeat'
259
- config.handler :server, MessageDispatcher
260
- end
261
- end
262
-
263
- def master_watcher
264
- @master_watcher ||= RedisWatcher.new(self)
265
- end
266
-
267
- def determine_initial_master
268
- if master_file_exists? && @current_master = redis_master_from_master_file
269
- if redis.slaves.include?(current_master)
270
- master_unavailable!
271
- elsif redis.unknowns.include?(current_master)
272
- master_unavailable!
273
- end
274
- else
275
- write_redis_master_file(current_master.server) if @current_master = redis.auto_detect_master
276
- end
277
- current_master or raise NoRedisMaster.new("Failed to determine initial redis master")
278
- end
279
-
280
- def determine_new_master
281
- redis.unknowns.include?(current_master) ? redis.slaves_of(current_master).first : current_master
282
- end
283
-
284
- def validate_pong_client_id(client_id)
285
- unless known_client = client_id_valid?(client_id)
286
- add_unknown_client_id(client_id)
287
- msg = "Received pong message from unknown id '#{client_id}'"
288
- logger.error msg
289
- logger.info "Sending system_notification message with text: #{msg}"
290
- beetle.publish(:system_notification, {"message" => msg}.to_json)
291
- end
292
- known_client
293
- end
294
-
295
- def client_id_valid?(client_id)
296
- @client_ids.include?(client_id)
297
- end
298
-
299
- def redeem_token(token)
300
- valid_token = token == @current_token
301
- logger.info "Ignored message (token was '#{token.inspect}', but expected '#{@current_token.inspect}')" unless valid_token
302
- valid_token
303
- end
304
-
305
- def start_invalidation
306
- @client_pong_ids_received.clear
307
- @client_invalidated_ids_received.clear
308
- check_all_clients_available
309
- end
310
-
311
- def check_all_clients_available
312
- generate_new_token
313
- logger.info "Sending ping message with token '#{@current_token}'"
314
- beetle.publish(:ping, payload_with_current_token)
315
- @available_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
316
- end
317
-
318
- def invalidate_current_master
319
- generate_new_token
320
- logger.info "Sending invalidate message with token '#{@current_token}'"
321
- beetle.publish(:invalidate, payload_with_current_token)
322
- @invalidate_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
323
- end
324
-
325
- def cancel_invalidation
326
- logger.warn "Redis master invalidation cancelled: 'pong' received from '#{@client_pong_ids_received.to_a.join(',')}', 'client_invalidated' received from '#{@client_invalidated_ids_received.to_a.join(',')}'"
327
- generate_new_token
328
- master_watcher.continue
329
- end
330
-
331
- def generate_new_token
332
- @current_token += 1
333
- end
334
-
335
- def payload_with_current_token(message = {})
336
- message["token"] = @current_token
337
- message.to_json
338
- end
339
-
340
- def all_client_pong_ids_received?
341
- @client_ids == @client_pong_ids_received
342
- end
343
-
344
- def all_client_invalidated_ids_received?
345
- @client_ids == @client_invalidated_ids_received
346
- end
347
-
348
- def switch_master
349
- if new_master = determine_new_master
350
- msg = "Setting redis master to '#{new_master.server}' (was '#{current_master.server}')"
351
- logger.warn msg
352
- logger.info "Sending system_notification message with text: #{msg}"
353
- beetle.publish(:system_notification, {"message" => msg}.to_json)
354
-
355
- new_master.master!
356
- write_redis_master_file(new_master.server)
357
- @current_master = new_master
358
- else
359
- msg = "Redis master could not be switched, no slave available to become new master, promoting old master"
360
- logger.error msg
361
- logger.info "Sending system_notification message with text: #{msg}"
362
- beetle.publish(:system_notification, {"message" => msg}.to_json)
363
- end
364
-
365
- publish_master(current_master)
366
- master_watcher.continue
367
- end
368
-
369
- def publish_master(master)
370
- logger.info "Sending reconfigure message with server '#{master.server}'"
371
- beetle.publish(:reconfigure, payload_with_current_token({"server" => master.server}))
372
- end
373
-
374
- def configure_slaves(master)
375
- (redis.masters-[master]).each do |r|
376
- logger.info "Reconfiguring '#{r.server}' as a slave of '#{master.server}'"
377
- r.slave_of!(master.host, master.port)
378
- end
379
- end
380
-
381
- # Periodically checks a redis server for availability
382
- class RedisWatcher #:nodoc:
383
- include Logging
384
-
385
- def initialize(configuration_server)
386
- @configuration_server = configuration_server
387
- @retries = 0
388
- @paused = true
389
- @master_retry_interval = configuration_server.config.redis_configuration_master_retry_interval
390
- @master_retries = configuration_server.config.redis_configuration_master_retries
391
- end
392
-
393
- def pause
394
- logger.info "Pause checking availability of redis servers"
395
- @watch_timer.cancel if @watch_timer
396
- @watch_timer = nil
397
- @paused = true
398
- end
399
-
400
- def watch
401
- @watch_timer ||=
402
- begin
403
- logger.info "Start watching redis servers every #{@master_retry_interval} seconds"
404
- EventMachine::add_periodic_timer(@master_retry_interval) { check_availability }
405
- end
406
- @paused = false
407
- end
408
- alias continue watch
409
-
410
- def paused?
411
- @paused
412
- end
413
-
414
- private
415
-
416
- def check_availability
417
- @configuration_server.redis.refresh
418
- if @configuration_server.master_available?
419
- @configuration_server.master_available!
420
- else
421
- logger.warn "Redis master not available! (Retries left: #{@master_retries - (@retries + 1)})"
422
- if (@retries+=1) >= @master_retries
423
- @retries = 0
424
- @configuration_server.master_unavailable!
425
- end
426
- end
427
- end
428
- end
429
-
430
- # Dispatches messages from the queue to methods in RedisConfigurationServer
431
- class MessageDispatcher < Beetle::Handler #:nodoc:
432
- cattr_accessor :configuration_server
433
- def process
434
- @@configuration_server.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
435
- end
436
- end
437
- end
438
- end
@@ -1,66 +0,0 @@
1
- module Beetle
2
- # class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
3
- class RedisServerInfo
4
- include Logging
5
-
6
- def initialize(config, options) #:nodoc:
7
- @config = config
8
- @options = options
9
- reset
10
- end
11
-
12
- # all configured redis servers
13
- def instances
14
- @instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
15
- end
16
-
17
- # fetches the server from the instances whith the given <tt>server</tt> string
18
- def find(server)
19
- instances.find{|r| r.server == server}
20
- end
21
-
22
- # refresh connectivity/role information
23
- def refresh
24
- logger.debug "Updating redis server info"
25
- reset
26
- instances.each {|r| @server_info[r.role] << r}
27
- end
28
-
29
- # subset of instances which are masters
30
- def masters
31
- @server_info["master"]
32
- end
33
-
34
- # subset of instances which are slaves
35
- def slaves
36
- @server_info["slave"]
37
- end
38
-
39
- # subset of instances which are not reachable
40
- def unknowns
41
- @server_info["unknown"]
42
- end
43
-
44
- # subset of instances which are set up as slaves of the given <tt>master</tt>
45
- def slaves_of(master)
46
- slaves.select{|r| r.slave_of?(master.host, master.port)}
47
- end
48
-
49
- # determine a master if we have one master and all other servers are slaves
50
- def auto_detect_master
51
- return nil unless master_and_slaves_reachable?
52
- masters.first
53
- end
54
-
55
- # check whether all redis servers are up and configured
56
- def master_and_slaves_reachable?
57
- masters.size == 1 && slaves.size == instances.size - 1
58
- end
59
-
60
- private
61
-
62
- def reset
63
- @server_info = Hash.new {|h,k| h[k]= []}
64
- end
65
- end
66
- end