beetle 0.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/README.rdoc +18 -8
  2. data/beetle.gemspec +37 -121
  3. data/bin/beetle +9 -0
  4. data/examples/README.rdoc +0 -2
  5. data/examples/rpc.rb +3 -2
  6. data/ext/mkrf_conf.rb +19 -0
  7. data/lib/beetle/base.rb +1 -8
  8. data/lib/beetle/client.rb +16 -14
  9. data/lib/beetle/commands/configuration_client.rb +73 -0
  10. data/lib/beetle/commands/configuration_server.rb +85 -0
  11. data/lib/beetle/commands.rb +30 -0
  12. data/lib/beetle/configuration.rb +70 -7
  13. data/lib/beetle/deduplication_store.rb +50 -38
  14. data/lib/beetle/handler.rb +2 -5
  15. data/lib/beetle/logging.rb +7 -0
  16. data/lib/beetle/message.rb +11 -13
  17. data/lib/beetle/publisher.rb +2 -2
  18. data/lib/beetle/r_c.rb +2 -1
  19. data/lib/beetle/redis_configuration_client.rb +136 -0
  20. data/lib/beetle/redis_configuration_server.rb +301 -0
  21. data/lib/beetle/redis_ext.rb +79 -0
  22. data/lib/beetle/redis_master_file.rb +35 -0
  23. data/lib/beetle/redis_server_info.rb +65 -0
  24. data/lib/beetle/subscriber.rb +4 -1
  25. data/lib/beetle.rb +2 -2
  26. data/test/beetle/configuration_test.rb +14 -2
  27. data/test/beetle/deduplication_store_test.rb +61 -43
  28. data/test/beetle/message_test.rb +28 -4
  29. data/test/beetle/redis_configuration_client_test.rb +97 -0
  30. data/test/beetle/redis_configuration_server_test.rb +278 -0
  31. data/test/beetle/redis_ext_test.rb +71 -0
  32. data/test/beetle/redis_master_file_test.rb +39 -0
  33. data/test/test_helper.rb +13 -1
  34. metadata +59 -50
  35. data/.gitignore +0 -5
  36. data/MIT-LICENSE +0 -20
  37. data/Rakefile +0 -114
  38. data/TODO +0 -7
  39. data/doc/redundant_queues.graffle +0 -7744
  40. data/etc/redis-master.conf +0 -189
  41. data/etc/redis-slave.conf +0 -189
  42. data/examples/redis_failover.rb +0 -65
  43. data/script/start_rabbit +0 -29
  44. data/snafu.rb +0 -55
  45. data/test/beetle/bla.rb +0 -0
  46. data/test/beetle.yml +0 -81
  47. data/tmp/master/.gitignore +0 -2
  48. data/tmp/slave/.gitignore +0 -3
@@ -11,15 +11,23 @@ module Beetle
11
11
  #
12
12
  # It also provides a method to garbage collect keys for expired messages.
13
13
  class DeduplicationStore
14
- # creates a new deduplication store
15
- def initialize(hosts = "localhost:6379", db = 4)
16
- @hosts = hosts
17
- @db = db
14
+ include Logging
15
+
16
+ def initialize(config = Beetle.config)
17
+ @config = config
18
+ @current_master = nil
19
+ @last_time_master_file_changed = nil
18
20
  end
19
21
 
20
22
  # get the Redis instance
21
23
  def redis
22
- @redis ||= find_redis_master
24
+ redis_master_source = @config.redis_server =~ /^\S+\:\d+$/ ? "server_string" : "master_file"
25
+ _eigenclass_.class_eval <<-EVALS, __FILE__, __LINE__
26
+ def redis
27
+ redis_master_from_#{redis_master_source}
28
+ end
29
+ EVALS
30
+ redis
23
31
  end
24
32
 
25
33
  # list of key suffixes to use for storing values in Redis.
@@ -43,7 +51,7 @@ module Beetle
43
51
  # garbage collect keys in Redis (always assume the worst!)
44
52
  def garbage_collect_keys(now = Time.now.to_i)
45
53
  keys = redis.keys("msgid:*:expires")
46
- threshold = now + Beetle.config.gc_threshold
54
+ threshold = now + @config.gc_threshold
47
55
  keys.each do |key|
48
56
  expires_at = redis.get key
49
57
  if expires_at && expires_at.to_i < threshold
@@ -53,20 +61,20 @@ module Beetle
53
61
  end
54
62
  end
55
63
 
56
- # unconditionally store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt>.
64
+ # unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
57
65
  def set(msg_id, suffix, value)
58
66
  with_failover { redis.set(key(msg_id, suffix), value) }
59
67
  end
60
68
 
61
- # store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
69
+ # store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
62
70
  def setnx(msg_id, suffix, value)
63
71
  with_failover { redis.setnx(key(msg_id, suffix), value) }
64
72
  end
65
73
 
66
74
  # store some key/value pairs if none of the given keys exist.
67
75
  def msetnx(msg_id, values)
68
- values = values.inject({}){|h,(k,v)| h[key(msg_id, k)] = v; h}
69
- with_failover { redis.msetnx(values) }
76
+ values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
77
+ with_failover { redis.msetnx(*values) }
70
78
  end
71
79
 
72
80
  # increment counter for key with given <tt>suffix</tt> for given <tt>msg_id</tt>. returns an integer.
@@ -86,7 +94,7 @@ module Beetle
86
94
 
87
95
  # delete all keys associated with the given <tt>msg_id</tt>.
88
96
  def del_keys(msg_id)
89
- with_failover { redis.del(keys(msg_id)) }
97
+ with_failover { redis.del(*keys(msg_id)) }
90
98
  end
91
99
 
92
100
  # check whether key with given suffix exists for a given <tt>msg_id</tt>.
@@ -101,16 +109,15 @@ module Beetle
101
109
 
102
110
  # performs redis operations by yielding a passed in block, waiting for a new master to
103
111
  # show up on the network if the operation throws an exception. if a new master doesn't
104
- # appear after 120 seconds, we raise an exception.
112
+ # appear after the configured timeout interval, we raise an exception.
105
113
  def with_failover #:nodoc:
106
- tries = 0
114
+ end_time = Time.now.to_i + @config.redis_failover_timeout.to_i
107
115
  begin
108
116
  yield
109
117
  rescue Exception => e
110
118
  Beetle::reraise_expectation_errors!
111
- logger.error "Beetle: redis connection error '#{e}'"
112
- if (tries+=1) < 120
113
- @redis = nil
119
+ logger.error "Beetle: redis connection error #{e} #{@config.redis_server} (#{e.backtrace[0]})"
120
+ if Time.now.to_i < end_time
114
121
  sleep 1
115
122
  logger.info "Beetle: retrying redis operation"
116
123
  retry
@@ -120,33 +127,38 @@ module Beetle
120
127
  end
121
128
  end
122
129
 
123
- # find the master redis instance
124
- def find_redis_master
125
- masters = []
126
- redis_instances.each do |redis|
127
- begin
128
- masters << redis if redis.info[:role] == "master"
129
- rescue Exception => e
130
- logger.error "Beetle: could not determine status of redis instance #{redis.server}"
131
- end
132
- end
133
- raise NoRedisMaster.new("unable to determine a new master redis instance") if masters.empty?
134
- raise TwoRedisMasters.new("more than one redis master instances") if masters.size > 1
135
- logger.info "Beetle: configured new redis master #{masters.first.server}"
136
- masters.first
130
+ # set current redis master instance (as specified in the Beetle::Configuration)
131
+ def redis_master_from_server_string
132
+ @current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db)
137
133
  end
138
134
 
139
- # returns the list of redis instances
140
- def redis_instances
141
- @redis_instances ||= @hosts.split(/ *, */).map{|s| s.split(':')}.map do |host, port|
142
- Redis.new(:host => host, :port => port, :db => @db)
143
- end
135
+ # set current redis master from master file
136
+ def redis_master_from_master_file
137
+ set_current_redis_master_from_master_file if redis_master_file_changed?
138
+ @current_master
139
+ rescue Errno::ENOENT
140
+ nil
144
141
  end
145
142
 
146
- # returns the configured logger
147
- def logger
148
- Beetle.config.logger
143
+ # redis master file changed outside the running process?
144
+ def redis_master_file_changed?
145
+ @last_time_master_file_changed != File.mtime(@config.redis_server)
149
146
  end
150
147
 
148
+ # set current redis master from server:port string contained in the redis master file
149
+ def set_current_redis_master_from_master_file
150
+ @last_time_master_file_changed = File.mtime(@config.redis_server)
151
+ server_string = read_master_file
152
+ @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
153
+ end
154
+
155
+ # server:port string from the redis master file
156
+ def read_master_file
157
+ File.read(@config.redis_server).chomp
158
+ end
159
+
160
+ def _eigenclass_ #:nodoc:
161
+ class << self; self; end
162
+ end
151
163
  end
152
164
  end
@@ -6,6 +6,8 @@ module Beetle
6
6
  # Most applications will define Handler subclasses and override the process, error and
7
7
  # failure methods.
8
8
  class Handler
9
+ include Logging
10
+
9
11
  # the Message instance which caused the handler to be created
10
12
  attr_reader :message
11
13
 
@@ -81,11 +83,6 @@ module Beetle
81
83
  logger.error "Beetle: handler has finally failed"
82
84
  end
83
85
 
84
- # returns the configured Beetle logger
85
- def logger
86
- Beetle.config.logger
87
- end
88
-
89
86
  # returns the configured Beetle logger
90
87
  def self.logger
91
88
  Beetle.config.logger
@@ -0,0 +1,7 @@
1
+ module Beetle
2
+ module Logging
3
+ def logger
4
+ Beetle.config.logger
5
+ end
6
+ end
7
+ end
@@ -6,6 +6,8 @@ module Beetle
6
6
  # should retry executing the message handler after a handler has crashed (or forcefully
7
7
  # aborted).
8
8
  class Message
9
+ include Logging
10
+
9
11
  # current message format version
10
12
  FORMAT_VERSION = 1
11
13
  # flag for encoding redundant messages
@@ -14,7 +16,7 @@ module Beetle
14
16
  DEFAULT_TTL = 1.day
15
17
  # forcefully abort a running handler after this many seconds.
16
18
  # can be overriden when registering a handler.
17
- DEFAULT_HANDLER_TIMEOUT = 300.seconds
19
+ DEFAULT_HANDLER_TIMEOUT = 600.seconds
18
20
  # how many times we should try to run a handler before giving up
19
21
  DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
20
22
  # how many seconds we should wait before retrying handler execution
@@ -77,6 +79,8 @@ module Beetle
77
79
  @format_version = headers[:format_version].to_i
78
80
  @flags = headers[:flags].to_i
79
81
  @expires_at = headers[:expires_at].to_i
82
+ rescue Exception => @exception
83
+ logger.error "Could not decode message. #{self.inspect}"
80
84
  end
81
85
 
82
86
  # build hash with options for the publisher
@@ -193,7 +197,7 @@ module Beetle
193
197
  # have we already seen this message? if not, set the status to "incomplete" and store
194
198
  # the message exipration timestamp in the deduplication store.
195
199
  def key_exists?
196
- old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at)
200
+ old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at, :timeout => now + timeout)
197
201
  if old_message
198
202
  logger.debug "Beetle: received duplicate message: #{msg_id} on queue: #{@queue}"
199
203
  end
@@ -236,7 +240,10 @@ module Beetle
236
240
  private
237
241
 
238
242
  def process_internal(handler)
239
- if expired?
243
+ if @exception
244
+ ack!
245
+ RC::DecodingError
246
+ elsif expired?
240
247
  logger.warn "Beetle: ignored expired message (#{msg_id})!"
241
248
  ack!
242
249
  RC::Ancient
@@ -244,7 +251,6 @@ module Beetle
244
251
  ack!
245
252
  run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
246
253
  elsif !key_exists?
247
- set_timeout!
248
254
  run_handler!(handler)
249
255
  elsif completed?
250
256
  ack!
@@ -273,7 +279,7 @@ module Beetle
273
279
  end
274
280
 
275
281
  def run_handler(handler)
276
- Timeout::timeout(@timeout) { @handler_result = handler.call(self) }
282
+ Redis::Timer.timeout(@timeout.to_f) { @handler_result = handler.call(self) }
277
283
  RC::OK
278
284
  rescue Exception => @exception
279
285
  Beetle::reraise_expectation_errors!
@@ -313,14 +319,6 @@ module Beetle
313
319
  end
314
320
  end
315
321
 
316
- def logger
317
- @logger ||= self.class.logger
318
- end
319
-
320
- def self.logger
321
- Beetle.config.logger
322
- end
323
-
324
322
  # ack the message for rabbit. deletes all keys associated with this message in the
325
323
  # deduplication store if we are sure this is the last message with the given msg_id.
326
324
  def ack!
@@ -165,9 +165,9 @@ module Beetle
165
165
 
166
166
  # TODO: Refactor, fethch the keys and stuff itself
167
167
  def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
168
- logger.debug("Creating queue with opts: #{creation_keys.inspect}")
168
+ logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
169
169
  queue = bunny.queue(queue_name, creation_keys)
170
- logger.debug("Binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
170
+ logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
171
171
  queue.bind(exchange(exchange_name), binding_keys)
172
172
  queue
173
173
  end
data/lib/beetle/r_c.rb CHANGED
@@ -27,7 +27,7 @@ module Beetle
27
27
  end
28
28
 
29
29
  rc :OK
30
- rc :Ancient, :failure
30
+ rc :Ancient
31
31
  rc :AttemptsLimitReached, :failure
32
32
  rc :ExceptionsLimitReached, :failure
33
33
  rc :Delayed, :recover
@@ -35,6 +35,7 @@ module Beetle
35
35
  rc :HandlerNotYetTimedOut, :recover
36
36
  rc :MutexLocked, :recover
37
37
  rc :InternalError, :recover
38
+ rc :DecodingError, :failure
38
39
 
39
40
  end
40
41
  end
@@ -0,0 +1,136 @@
1
+ module Beetle
2
+ # A RedisConfigurationClient is the subordinate part of beetle's
3
+ # redis failover solution
4
+ #
5
+ # An instance of RedisConfigurationClient lives on every server that
6
+ # hosts message consumers (worker server).
7
+ #
8
+ # It is responsible for determining an initial redis master and reacting to redis master
9
+ # switches initiated by the RedisConfigurationServer.
10
+ #
11
+ # It will write the current redis master host:port string to a file specified via a
12
+ # Configuration, which is then read by DeduplicationStore on redis access.
13
+ #
14
+ # Usually started via <tt>beetle configuration_client</tt> command.
15
+ class RedisConfigurationClient
16
+ include Logging
17
+ include RedisMasterFile
18
+
19
+ # Set a custom unique id for this instance. Must match an entry in
20
+ # Configuration#redis_configuration_client_ids.
21
+ attr_writer :id
22
+
23
+ # The current redis master
24
+ attr_reader :current_master
25
+
26
+ # Unique id for this instance (defaults to the hostname)
27
+ def id
28
+ @id ||= `hostname`.chomp
29
+ end
30
+
31
+ def initialize #:nodoc:
32
+ @current_token = nil
33
+ MessageDispatcher.configuration_client = self
34
+ end
35
+
36
+ # determinines the initial redis master (if possible), then enters a messaging event
37
+ # loop, reacting to failover related messages sent by RedisConfigurationServer.
38
+ def start
39
+ verify_redis_master_file_string
40
+ logger.info "RedisConfigurationClient starting (client id: #{id})"
41
+ determine_initial_master
42
+ clear_redis_master_file unless current_master.try(:master?)
43
+ logger.info "Listening"
44
+ beetle.listen
45
+ end
46
+
47
+ # called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
48
+ def ping(payload)
49
+ token = payload["token"]
50
+ logger.info "Received ping message with token '#{token}'"
51
+ pong! if redeem_token(token)
52
+ end
53
+
54
+ # called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
55
+ def invalidate(payload)
56
+ token = payload["token"]
57
+ logger.info "Received invalidate message with token '#{token}'"
58
+ invalidate! if redeem_token(token) && !current_master.try(:master?)
59
+ end
60
+
61
+ # called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
62
+ def reconfigure(payload)
63
+ server = payload["server"]
64
+ token = payload["token"]
65
+ logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
66
+ return unless redeem_token(token)
67
+ unless server == read_redis_master_file
68
+ new_master!(server)
69
+ write_redis_master_file(server)
70
+ end
71
+ end
72
+
73
+ # Beetle::Client instance for communication with the RedisConfigurationServer
74
+ def beetle
75
+ @beetle ||= build_beetle
76
+ end
77
+
78
+ def config #:nodoc:
79
+ beetle.config
80
+ end
81
+
82
+ private
83
+
84
+ def determine_initial_master
85
+ if master_file_exists? && server = read_redis_master_file
86
+ new_master!(server)
87
+ end
88
+ end
89
+
90
+ def new_master!(server)
91
+ @current_master = Redis.from_server_string(server, :timeout => 3)
92
+ end
93
+
94
+ def build_beetle
95
+ system = Beetle.config.system_name
96
+ Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
97
+ config.message :ping
98
+ config.queue :ping, :amqp_name => "#{system}_ping_#{id}"
99
+ config.message :pong
100
+ config.message :invalidate
101
+ config.queue :invalidate, :amqp_name => "#{system}_invalidate_#{id}"
102
+ config.message :client_invalidated
103
+ config.message :reconfigure
104
+ config.queue :reconfigure, :amqp_name => "#{system}_reconfigure_#{id}"
105
+
106
+ config.handler [:ping, :invalidate, :reconfigure], MessageDispatcher
107
+ end
108
+ end
109
+
110
+ def redeem_token(token)
111
+ @current_token = token if @current_token.nil? || token > @current_token
112
+ token_valid = token >= @current_token
113
+ logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
114
+ token_valid
115
+ end
116
+
117
+ def pong!
118
+ beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
119
+ end
120
+
121
+ def invalidate!
122
+ @current_master = nil
123
+ clear_redis_master_file
124
+ beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
125
+ end
126
+
127
+
128
+ # Dispatches messages from the queue to methods in RedisConfigurationClient
129
+ class MessageDispatcher < Beetle::Handler #:nodoc:
130
+ cattr_accessor :configuration_client
131
+ def process
132
+ @@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
133
+ end
134
+ end
135
+ end
136
+ end