beetle 0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +18 -8
- data/beetle.gemspec +37 -121
- data/bin/beetle +9 -0
- data/examples/README.rdoc +0 -2
- data/examples/rpc.rb +3 -2
- data/ext/mkrf_conf.rb +19 -0
- data/lib/beetle/base.rb +1 -8
- data/lib/beetle/client.rb +16 -14
- data/lib/beetle/commands/configuration_client.rb +73 -0
- data/lib/beetle/commands/configuration_server.rb +85 -0
- data/lib/beetle/commands.rb +30 -0
- data/lib/beetle/configuration.rb +70 -7
- data/lib/beetle/deduplication_store.rb +50 -38
- data/lib/beetle/handler.rb +2 -5
- data/lib/beetle/logging.rb +7 -0
- data/lib/beetle/message.rb +11 -13
- data/lib/beetle/publisher.rb +2 -2
- data/lib/beetle/r_c.rb +2 -1
- data/lib/beetle/redis_configuration_client.rb +136 -0
- data/lib/beetle/redis_configuration_server.rb +301 -0
- data/lib/beetle/redis_ext.rb +79 -0
- data/lib/beetle/redis_master_file.rb +35 -0
- data/lib/beetle/redis_server_info.rb +65 -0
- data/lib/beetle/subscriber.rb +4 -1
- data/lib/beetle.rb +2 -2
- data/test/beetle/configuration_test.rb +14 -2
- data/test/beetle/deduplication_store_test.rb +61 -43
- data/test/beetle/message_test.rb +28 -4
- data/test/beetle/redis_configuration_client_test.rb +97 -0
- data/test/beetle/redis_configuration_server_test.rb +278 -0
- data/test/beetle/redis_ext_test.rb +71 -0
- data/test/beetle/redis_master_file_test.rb +39 -0
- data/test/test_helper.rb +13 -1
- metadata +59 -50
- data/.gitignore +0 -5
- data/MIT-LICENSE +0 -20
- data/Rakefile +0 -114
- data/TODO +0 -7
- data/doc/redundant_queues.graffle +0 -7744
- data/etc/redis-master.conf +0 -189
- data/etc/redis-slave.conf +0 -189
- data/examples/redis_failover.rb +0 -65
- data/script/start_rabbit +0 -29
- data/snafu.rb +0 -55
- data/test/beetle/bla.rb +0 -0
- data/test/beetle.yml +0 -81
- data/tmp/master/.gitignore +0 -2
- data/tmp/slave/.gitignore +0 -3
@@ -11,15 +11,23 @@ module Beetle
|
|
11
11
|
#
|
12
12
|
# It also provides a method to garbage collect keys for expired messages.
|
13
13
|
class DeduplicationStore
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
@
|
14
|
+
include Logging
|
15
|
+
|
16
|
+
def initialize(config = Beetle.config)
|
17
|
+
@config = config
|
18
|
+
@current_master = nil
|
19
|
+
@last_time_master_file_changed = nil
|
18
20
|
end
|
19
21
|
|
20
22
|
# get the Redis instance
|
21
23
|
def redis
|
22
|
-
@
|
24
|
+
redis_master_source = @config.redis_server =~ /^\S+\:\d+$/ ? "server_string" : "master_file"
|
25
|
+
_eigenclass_.class_eval <<-EVALS, __FILE__, __LINE__
|
26
|
+
def redis
|
27
|
+
redis_master_from_#{redis_master_source}
|
28
|
+
end
|
29
|
+
EVALS
|
30
|
+
redis
|
23
31
|
end
|
24
32
|
|
25
33
|
# list of key suffixes to use for storing values in Redis.
|
@@ -43,7 +51,7 @@ module Beetle
|
|
43
51
|
# garbage collect keys in Redis (always assume the worst!)
|
44
52
|
def garbage_collect_keys(now = Time.now.to_i)
|
45
53
|
keys = redis.keys("msgid:*:expires")
|
46
|
-
threshold = now +
|
54
|
+
threshold = now + @config.gc_threshold
|
47
55
|
keys.each do |key|
|
48
56
|
expires_at = redis.get key
|
49
57
|
if expires_at && expires_at.to_i < threshold
|
@@ -53,20 +61,20 @@ module Beetle
|
|
53
61
|
end
|
54
62
|
end
|
55
63
|
|
56
|
-
# unconditionally store a key
|
64
|
+
# unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
|
57
65
|
def set(msg_id, suffix, value)
|
58
66
|
with_failover { redis.set(key(msg_id, suffix), value) }
|
59
67
|
end
|
60
68
|
|
61
|
-
# store a key
|
69
|
+
# store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
|
62
70
|
def setnx(msg_id, suffix, value)
|
63
71
|
with_failover { redis.setnx(key(msg_id, suffix), value) }
|
64
72
|
end
|
65
73
|
|
66
74
|
# store some key/value pairs if none of the given keys exist.
|
67
75
|
def msetnx(msg_id, values)
|
68
|
-
values = values.inject(
|
69
|
-
with_failover { redis.msetnx(values) }
|
76
|
+
values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
|
77
|
+
with_failover { redis.msetnx(*values) }
|
70
78
|
end
|
71
79
|
|
72
80
|
# increment counter for key with given <tt>suffix</tt> for given <tt>msg_id</tt>. returns an integer.
|
@@ -86,7 +94,7 @@ module Beetle
|
|
86
94
|
|
87
95
|
# delete all keys associated with the given <tt>msg_id</tt>.
|
88
96
|
def del_keys(msg_id)
|
89
|
-
with_failover { redis.del(keys(msg_id)) }
|
97
|
+
with_failover { redis.del(*keys(msg_id)) }
|
90
98
|
end
|
91
99
|
|
92
100
|
# check whether key with given suffix exists for a given <tt>msg_id</tt>.
|
@@ -101,16 +109,15 @@ module Beetle
|
|
101
109
|
|
102
110
|
# performs redis operations by yielding a passed in block, waiting for a new master to
|
103
111
|
# show up on the network if the operation throws an exception. if a new master doesn't
|
104
|
-
# appear after
|
112
|
+
# appear after the configured timeout interval, we raise an exception.
|
105
113
|
def with_failover #:nodoc:
|
106
|
-
|
114
|
+
end_time = Time.now.to_i + @config.redis_failover_timeout.to_i
|
107
115
|
begin
|
108
116
|
yield
|
109
117
|
rescue Exception => e
|
110
118
|
Beetle::reraise_expectation_errors!
|
111
|
-
logger.error "Beetle: redis connection error
|
112
|
-
if
|
113
|
-
@redis = nil
|
119
|
+
logger.error "Beetle: redis connection error #{e} #{@config.redis_server} (#{e.backtrace[0]})"
|
120
|
+
if Time.now.to_i < end_time
|
114
121
|
sleep 1
|
115
122
|
logger.info "Beetle: retrying redis operation"
|
116
123
|
retry
|
@@ -120,33 +127,38 @@ module Beetle
|
|
120
127
|
end
|
121
128
|
end
|
122
129
|
|
123
|
-
#
|
124
|
-
def
|
125
|
-
|
126
|
-
redis_instances.each do |redis|
|
127
|
-
begin
|
128
|
-
masters << redis if redis.info[:role] == "master"
|
129
|
-
rescue Exception => e
|
130
|
-
logger.error "Beetle: could not determine status of redis instance #{redis.server}"
|
131
|
-
end
|
132
|
-
end
|
133
|
-
raise NoRedisMaster.new("unable to determine a new master redis instance") if masters.empty?
|
134
|
-
raise TwoRedisMasters.new("more than one redis master instances") if masters.size > 1
|
135
|
-
logger.info "Beetle: configured new redis master #{masters.first.server}"
|
136
|
-
masters.first
|
130
|
+
# set current redis master instance (as specified in the Beetle::Configuration)
|
131
|
+
def redis_master_from_server_string
|
132
|
+
@current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db)
|
137
133
|
end
|
138
134
|
|
139
|
-
#
|
140
|
-
def
|
141
|
-
|
142
|
-
|
143
|
-
|
135
|
+
# set current redis master from master file
|
136
|
+
def redis_master_from_master_file
|
137
|
+
set_current_redis_master_from_master_file if redis_master_file_changed?
|
138
|
+
@current_master
|
139
|
+
rescue Errno::ENOENT
|
140
|
+
nil
|
144
141
|
end
|
145
142
|
|
146
|
-
#
|
147
|
-
def
|
148
|
-
|
143
|
+
# redis master file changed outside the running process?
|
144
|
+
def redis_master_file_changed?
|
145
|
+
@last_time_master_file_changed != File.mtime(@config.redis_server)
|
149
146
|
end
|
150
147
|
|
148
|
+
# set current redis master from server:port string contained in the redis master file
|
149
|
+
def set_current_redis_master_from_master_file
|
150
|
+
@last_time_master_file_changed = File.mtime(@config.redis_server)
|
151
|
+
server_string = read_master_file
|
152
|
+
@current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
|
153
|
+
end
|
154
|
+
|
155
|
+
# server:port string from the redis master file
|
156
|
+
def read_master_file
|
157
|
+
File.read(@config.redis_server).chomp
|
158
|
+
end
|
159
|
+
|
160
|
+
def _eigenclass_ #:nodoc:
|
161
|
+
class << self; self; end
|
162
|
+
end
|
151
163
|
end
|
152
164
|
end
|
data/lib/beetle/handler.rb
CHANGED
@@ -6,6 +6,8 @@ module Beetle
|
|
6
6
|
# Most applications will define Handler subclasses and override the process, error and
|
7
7
|
# failure methods.
|
8
8
|
class Handler
|
9
|
+
include Logging
|
10
|
+
|
9
11
|
# the Message instance which caused the handler to be created
|
10
12
|
attr_reader :message
|
11
13
|
|
@@ -81,11 +83,6 @@ module Beetle
|
|
81
83
|
logger.error "Beetle: handler has finally failed"
|
82
84
|
end
|
83
85
|
|
84
|
-
# returns the configured Beetle logger
|
85
|
-
def logger
|
86
|
-
Beetle.config.logger
|
87
|
-
end
|
88
|
-
|
89
86
|
# returns the configured Beetle logger
|
90
87
|
def self.logger
|
91
88
|
Beetle.config.logger
|
data/lib/beetle/message.rb
CHANGED
@@ -6,6 +6,8 @@ module Beetle
|
|
6
6
|
# should retry executing the message handler after a handler has crashed (or forcefully
|
7
7
|
# aborted).
|
8
8
|
class Message
|
9
|
+
include Logging
|
10
|
+
|
9
11
|
# current message format version
|
10
12
|
FORMAT_VERSION = 1
|
11
13
|
# flag for encoding redundant messages
|
@@ -14,7 +16,7 @@ module Beetle
|
|
14
16
|
DEFAULT_TTL = 1.day
|
15
17
|
# forcefully abort a running handler after this many seconds.
|
16
18
|
# can be overriden when registering a handler.
|
17
|
-
DEFAULT_HANDLER_TIMEOUT =
|
19
|
+
DEFAULT_HANDLER_TIMEOUT = 600.seconds
|
18
20
|
# how many times we should try to run a handler before giving up
|
19
21
|
DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
|
20
22
|
# how many seconds we should wait before retrying handler execution
|
@@ -77,6 +79,8 @@ module Beetle
|
|
77
79
|
@format_version = headers[:format_version].to_i
|
78
80
|
@flags = headers[:flags].to_i
|
79
81
|
@expires_at = headers[:expires_at].to_i
|
82
|
+
rescue Exception => @exception
|
83
|
+
logger.error "Could not decode message. #{self.inspect}"
|
80
84
|
end
|
81
85
|
|
82
86
|
# build hash with options for the publisher
|
@@ -193,7 +197,7 @@ module Beetle
|
|
193
197
|
# have we already seen this message? if not, set the status to "incomplete" and store
|
194
198
|
# the message exipration timestamp in the deduplication store.
|
195
199
|
def key_exists?
|
196
|
-
old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at)
|
200
|
+
old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at, :timeout => now + timeout)
|
197
201
|
if old_message
|
198
202
|
logger.debug "Beetle: received duplicate message: #{msg_id} on queue: #{@queue}"
|
199
203
|
end
|
@@ -236,7 +240,10 @@ module Beetle
|
|
236
240
|
private
|
237
241
|
|
238
242
|
def process_internal(handler)
|
239
|
-
if
|
243
|
+
if @exception
|
244
|
+
ack!
|
245
|
+
RC::DecodingError
|
246
|
+
elsif expired?
|
240
247
|
logger.warn "Beetle: ignored expired message (#{msg_id})!"
|
241
248
|
ack!
|
242
249
|
RC::Ancient
|
@@ -244,7 +251,6 @@ module Beetle
|
|
244
251
|
ack!
|
245
252
|
run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
|
246
253
|
elsif !key_exists?
|
247
|
-
set_timeout!
|
248
254
|
run_handler!(handler)
|
249
255
|
elsif completed?
|
250
256
|
ack!
|
@@ -273,7 +279,7 @@ module Beetle
|
|
273
279
|
end
|
274
280
|
|
275
281
|
def run_handler(handler)
|
276
|
-
|
282
|
+
Redis::Timer.timeout(@timeout.to_f) { @handler_result = handler.call(self) }
|
277
283
|
RC::OK
|
278
284
|
rescue Exception => @exception
|
279
285
|
Beetle::reraise_expectation_errors!
|
@@ -313,14 +319,6 @@ module Beetle
|
|
313
319
|
end
|
314
320
|
end
|
315
321
|
|
316
|
-
def logger
|
317
|
-
@logger ||= self.class.logger
|
318
|
-
end
|
319
|
-
|
320
|
-
def self.logger
|
321
|
-
Beetle.config.logger
|
322
|
-
end
|
323
|
-
|
324
322
|
# ack the message for rabbit. deletes all keys associated with this message in the
|
325
323
|
# deduplication store if we are sure this is the last message with the given msg_id.
|
326
324
|
def ack!
|
data/lib/beetle/publisher.rb
CHANGED
@@ -165,9 +165,9 @@ module Beetle
|
|
165
165
|
|
166
166
|
# TODO: Refactor, fethch the keys and stuff itself
|
167
167
|
def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
|
168
|
-
logger.debug("
|
168
|
+
logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
|
169
169
|
queue = bunny.queue(queue_name, creation_keys)
|
170
|
-
logger.debug("
|
170
|
+
logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
|
171
171
|
queue.bind(exchange(exchange_name), binding_keys)
|
172
172
|
queue
|
173
173
|
end
|
data/lib/beetle/r_c.rb
CHANGED
@@ -27,7 +27,7 @@ module Beetle
|
|
27
27
|
end
|
28
28
|
|
29
29
|
rc :OK
|
30
|
-
rc :Ancient
|
30
|
+
rc :Ancient
|
31
31
|
rc :AttemptsLimitReached, :failure
|
32
32
|
rc :ExceptionsLimitReached, :failure
|
33
33
|
rc :Delayed, :recover
|
@@ -35,6 +35,7 @@ module Beetle
|
|
35
35
|
rc :HandlerNotYetTimedOut, :recover
|
36
36
|
rc :MutexLocked, :recover
|
37
37
|
rc :InternalError, :recover
|
38
|
+
rc :DecodingError, :failure
|
38
39
|
|
39
40
|
end
|
40
41
|
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
module Beetle
|
2
|
+
# A RedisConfigurationClient is the subordinate part of beetle's
|
3
|
+
# redis failover solution
|
4
|
+
#
|
5
|
+
# An instance of RedisConfigurationClient lives on every server that
|
6
|
+
# hosts message consumers (worker server).
|
7
|
+
#
|
8
|
+
# It is responsible for determining an initial redis master and reacting to redis master
|
9
|
+
# switches initiated by the RedisConfigurationServer.
|
10
|
+
#
|
11
|
+
# It will write the current redis master host:port string to a file specified via a
|
12
|
+
# Configuration, which is then read by DeduplicationStore on redis access.
|
13
|
+
#
|
14
|
+
# Usually started via <tt>beetle configuration_client</tt> command.
|
15
|
+
class RedisConfigurationClient
|
16
|
+
include Logging
|
17
|
+
include RedisMasterFile
|
18
|
+
|
19
|
+
# Set a custom unique id for this instance. Must match an entry in
|
20
|
+
# Configuration#redis_configuration_client_ids.
|
21
|
+
attr_writer :id
|
22
|
+
|
23
|
+
# The current redis master
|
24
|
+
attr_reader :current_master
|
25
|
+
|
26
|
+
# Unique id for this instance (defaults to the hostname)
|
27
|
+
def id
|
28
|
+
@id ||= `hostname`.chomp
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize #:nodoc:
|
32
|
+
@current_token = nil
|
33
|
+
MessageDispatcher.configuration_client = self
|
34
|
+
end
|
35
|
+
|
36
|
+
# determinines the initial redis master (if possible), then enters a messaging event
|
37
|
+
# loop, reacting to failover related messages sent by RedisConfigurationServer.
|
38
|
+
def start
|
39
|
+
verify_redis_master_file_string
|
40
|
+
logger.info "RedisConfigurationClient starting (client id: #{id})"
|
41
|
+
determine_initial_master
|
42
|
+
clear_redis_master_file unless current_master.try(:master?)
|
43
|
+
logger.info "Listening"
|
44
|
+
beetle.listen
|
45
|
+
end
|
46
|
+
|
47
|
+
# called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
|
48
|
+
def ping(payload)
|
49
|
+
token = payload["token"]
|
50
|
+
logger.info "Received ping message with token '#{token}'"
|
51
|
+
pong! if redeem_token(token)
|
52
|
+
end
|
53
|
+
|
54
|
+
# called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
|
55
|
+
def invalidate(payload)
|
56
|
+
token = payload["token"]
|
57
|
+
logger.info "Received invalidate message with token '#{token}'"
|
58
|
+
invalidate! if redeem_token(token) && !current_master.try(:master?)
|
59
|
+
end
|
60
|
+
|
61
|
+
# called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
|
62
|
+
def reconfigure(payload)
|
63
|
+
server = payload["server"]
|
64
|
+
token = payload["token"]
|
65
|
+
logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
|
66
|
+
return unless redeem_token(token)
|
67
|
+
unless server == read_redis_master_file
|
68
|
+
new_master!(server)
|
69
|
+
write_redis_master_file(server)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Beetle::Client instance for communication with the RedisConfigurationServer
|
74
|
+
def beetle
|
75
|
+
@beetle ||= build_beetle
|
76
|
+
end
|
77
|
+
|
78
|
+
def config #:nodoc:
|
79
|
+
beetle.config
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def determine_initial_master
|
85
|
+
if master_file_exists? && server = read_redis_master_file
|
86
|
+
new_master!(server)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def new_master!(server)
|
91
|
+
@current_master = Redis.from_server_string(server, :timeout => 3)
|
92
|
+
end
|
93
|
+
|
94
|
+
def build_beetle
|
95
|
+
system = Beetle.config.system_name
|
96
|
+
Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
|
97
|
+
config.message :ping
|
98
|
+
config.queue :ping, :amqp_name => "#{system}_ping_#{id}"
|
99
|
+
config.message :pong
|
100
|
+
config.message :invalidate
|
101
|
+
config.queue :invalidate, :amqp_name => "#{system}_invalidate_#{id}"
|
102
|
+
config.message :client_invalidated
|
103
|
+
config.message :reconfigure
|
104
|
+
config.queue :reconfigure, :amqp_name => "#{system}_reconfigure_#{id}"
|
105
|
+
|
106
|
+
config.handler [:ping, :invalidate, :reconfigure], MessageDispatcher
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def redeem_token(token)
|
111
|
+
@current_token = token if @current_token.nil? || token > @current_token
|
112
|
+
token_valid = token >= @current_token
|
113
|
+
logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
|
114
|
+
token_valid
|
115
|
+
end
|
116
|
+
|
117
|
+
def pong!
|
118
|
+
beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
|
119
|
+
end
|
120
|
+
|
121
|
+
def invalidate!
|
122
|
+
@current_master = nil
|
123
|
+
clear_redis_master_file
|
124
|
+
beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
# Dispatches messages from the queue to methods in RedisConfigurationClient
|
129
|
+
class MessageDispatcher < Beetle::Handler #:nodoc:
|
130
|
+
cattr_accessor :configuration_client
|
131
|
+
def process
|
132
|
+
@@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|