beetle 1.0.4 → 2.0.0rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.rdoc +15 -7
- data/REDIS_AUTO_FAILOVER.rdoc +8 -16
- data/RELEASE_NOTES.rdoc +5 -0
- data/beetle.gemspec +22 -10
- data/features/support/system_notification_logger +29 -12
- data/features/support/test_daemons/redis.rb +1 -1
- data/features/support/test_daemons/redis_configuration_client.rb +3 -3
- data/features/support/test_daemons/redis_configuration_server.rb +2 -2
- data/lib/beetle/deduplication_store.rb +0 -66
- data/lib/beetle/redis_master_file.rb +0 -5
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +0 -48
- data/test/beetle/message_test.rb +0 -30
- metadata +165 -25
- data/bin/beetle +0 -9
- data/lib/beetle/commands/configuration_client.rb +0 -98
- data/lib/beetle/commands/configuration_server.rb +0 -98
- data/lib/beetle/commands/garbage_collect_deduplication_store.rb +0 -52
- data/lib/beetle/commands.rb +0 -35
- data/lib/beetle/redis_configuration_client.rb +0 -157
- data/lib/beetle/redis_configuration_http_server.rb +0 -152
- data/lib/beetle/redis_configuration_server.rb +0 -438
- data/lib/beetle/redis_server_info.rb +0 -66
- data/test/beetle/redis_configuration_client_test.rb +0 -118
- data/test/beetle/redis_configuration_server_test.rb +0 -381
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5648bfe9f8b8f76d05d04b360df2e160c7f9ab6b
|
4
|
+
data.tar.gz: 5fdb6ec4eea81aa6e7d79bbbdd9ed820711bdb2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e5e5f2ba5cb31942fbf01584cae8b448cec7db14e260050e74fb5765c9d29a23c9f7d1cd8e93cf2be04ea74d46739e57c45149fe9c4557e3fce77936814d6ed5
|
7
|
+
data.tar.gz: 28b1e12e6fc4b9858c6184ff93dbc876cc51ced13cac9f31664547b07fbf9dfa5557c83cc7d42a609cda20bee8f00b85d6a9520aa28290ab7172a141eaa7a93f
|
data/README.rdoc
CHANGED
@@ -29,10 +29,10 @@ More information can be found on the {project website}[http://xing.github.com/be
|
|
29
29
|
|
30
30
|
# configure exchanges, queues, bindings, messages and handlers
|
31
31
|
|
32
|
-
b.configure do
|
33
|
-
|
34
|
-
|
35
|
-
|
32
|
+
b.configure do
|
33
|
+
queue :test
|
34
|
+
message :test
|
35
|
+
handler(:test) { |message| puts message.data }
|
36
36
|
end
|
37
37
|
|
38
38
|
=== Publishing
|
@@ -72,6 +72,13 @@ You also need a Redis instance running. The default configuration of Redis will
|
|
72
72
|
|
73
73
|
redis-server
|
74
74
|
|
75
|
+
If you want to run the integration tests you need GO installed and you
|
76
|
+
will need to build the beetle binary. We provide a Makefile for this
|
77
|
+
purpose, so simply running
|
78
|
+
|
79
|
+
make
|
80
|
+
|
81
|
+
should suffice.
|
75
82
|
|
76
83
|
== Gem Dependencies
|
77
84
|
|
@@ -127,8 +134,9 @@ Update RELEASE_NOTES.rdoc!
|
|
127
134
|
We use {semantic versioning}[http://semver.org/] and create a git tag
|
128
135
|
for each release.
|
129
136
|
|
130
|
-
Edit `lib/beetle/version.rb`
|
131
|
-
|
137
|
+
Edit `lib/beetle/version.rb` and
|
138
|
+
`go/src/github.com/xing/beetle/version.go` to set the new version
|
139
|
+
number (`Major.Minor.Patch`).
|
132
140
|
|
133
141
|
In short (see {semver.org}[http://semver.org] for details):
|
134
142
|
|
@@ -141,7 +149,7 @@ In short (see {semver.org}[http://semver.org] for details):
|
|
141
149
|
are introduced.
|
142
150
|
|
143
151
|
Then use `rake release` which will create the git tag and upload the
|
144
|
-
gem to
|
152
|
+
gem to github.com:
|
145
153
|
|
146
154
|
$ bundle exec rake release
|
147
155
|
|
data/REDIS_AUTO_FAILOVER.rdoc
CHANGED
@@ -35,9 +35,9 @@ On every worker server runs another daemon, the Redis Configuration Client
|
|
35
35
|
|
36
36
|
If the RCS finds another potential Redis Master, it sends out a message to see
|
37
37
|
if all known RCCs are still available (once again to eliminate the risk of a
|
38
|
-
partitioned network) and if they agree to the master switch.
|
38
|
+
partitioned network) and if they agree to the master switch.
|
39
39
|
|
40
|
-
If all RCCs have answered to that message, the RCS sends out a message which
|
40
|
+
If all RCCs have answered to that message, the RCS sends out a message which
|
41
41
|
tells the RCCs to invalidate the current master.
|
42
42
|
|
43
43
|
This happens by deleting the contents of a special file which is used
|
@@ -53,7 +53,7 @@ to the RCCs, which then write that value into their redis master file.
|
|
53
53
|
|
54
54
|
Additionally, the RCS sends reconfigure messages with the current Redis master
|
55
55
|
periodically, to allow new RCCs to pick up the current master. Plus it turns
|
56
|
-
all other redis servers into slaves of the current master.
|
56
|
+
all other redis servers into slaves of the current master.
|
57
57
|
|
58
58
|
=== Prerequisites
|
59
59
|
|
@@ -93,15 +93,11 @@ All command line options can also be given as a yaml configuration file via the
|
|
93
93
|
|
94
94
|
Start the Redis Configuration Server:
|
95
95
|
|
96
|
-
beetle configuration_server
|
97
|
-
|
98
|
-
Get help for starting/stopping the server:
|
99
|
-
|
100
|
-
beetle configuration_server -h
|
96
|
+
beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
|
101
97
|
|
102
98
|
Get help for server options:
|
103
99
|
|
104
|
-
beetle configuration_server
|
100
|
+
beetle configuration_server -h
|
105
101
|
|
106
102
|
=== On every worker server
|
107
103
|
|
@@ -109,16 +105,12 @@ Start the Redis Configuration Client:
|
|
109
105
|
|
110
106
|
On first worker server:
|
111
107
|
|
112
|
-
beetle configuration_client
|
108
|
+
beetle configuration_client --client-id rcc-1
|
113
109
|
|
114
110
|
On second worker server:
|
115
111
|
|
116
|
-
beetle configuration_client
|
117
|
-
|
118
|
-
Get help for starting/stopping the client:
|
119
|
-
|
120
|
-
beetle configuration_client -h
|
112
|
+
beetle configuration_client --client-id rcc-2
|
121
113
|
|
122
114
|
Get help for client options:
|
123
115
|
|
124
|
-
beetle configuration_client
|
116
|
+
beetle configuration_client -h
|
data/RELEASE_NOTES.rdoc
CHANGED
data/beetle.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.description = "A highly available, reliable messaging infrastructure"
|
13
13
|
s.summary = "High Availability AMQP Messaging with Redundant Queues"
|
14
14
|
s.email = "opensource@xing.com"
|
15
|
-
s.executables = [
|
15
|
+
s.executables = []
|
16
16
|
s.extra_rdoc_files = Dir['**/*.rdoc'] + %w(MIT-LICENSE)
|
17
17
|
s.files = Dir['{examples,lib}/**/*.rb'] + Dir['{features,script}/**/*'] + %w(beetle.gemspec Rakefile)
|
18
18
|
s.homepage = "http://xing.github.com/beetle/"
|
@@ -21,13 +21,25 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.test_files = Dir['test/**/*.rb']
|
22
22
|
|
23
23
|
s.specification_version = 3
|
24
|
-
s.add_runtime_dependency
|
25
|
-
s.add_runtime_dependency
|
26
|
-
s.add_runtime_dependency
|
27
|
-
s.add_runtime_dependency
|
28
|
-
s.add_runtime_dependency
|
29
|
-
s.add_runtime_dependency
|
30
|
-
s.add_runtime_dependency
|
31
|
-
|
32
|
-
s.
|
24
|
+
s.add_runtime_dependency "uuid4r", ">= 0.1.2"
|
25
|
+
s.add_runtime_dependency "bunny", "~> 0.7.10"
|
26
|
+
s.add_runtime_dependency "redis", ">= 2.2.2"
|
27
|
+
s.add_runtime_dependency "hiredis", ">= 0.4.5"
|
28
|
+
s.add_runtime_dependency "amq-protocol", "= 2.0.1"
|
29
|
+
s.add_runtime_dependency "amqp", "= 1.6.0"
|
30
|
+
s.add_runtime_dependency "activesupport", ">= 2.3.4"
|
31
|
+
|
32
|
+
s.add_development_dependency "activerecord", "~> 5.0"
|
33
|
+
s.add_development_dependency "cucumber", "~> 2.4.0"
|
34
|
+
s.add_development_dependency "daemon_controller", "~> 1.2.0"
|
35
|
+
s.add_development_dependency "daemons", ">= 1.2.0"
|
36
|
+
s.add_development_dependency "i18n"
|
37
|
+
s.add_development_dependency "minitest", "~> 5.1"
|
38
|
+
s.add_development_dependency "mocha", "~> 1.1.0"
|
39
|
+
s.add_development_dependency "mysql2", "~> 0.4.4"
|
40
|
+
s.add_development_dependency "rake", "~> 11.2"
|
41
|
+
s.add_development_dependency "rdoc", "~> 4.0"
|
42
|
+
s.add_development_dependency "simplecov", "~> 0.12.0"
|
43
|
+
s.add_development_dependency "webmock", "~> 1.21.0"
|
44
|
+
s.add_development_dependency "websocket-eventmachine-client"
|
33
45
|
end
|
@@ -2,30 +2,47 @@
|
|
2
2
|
|
3
3
|
require "rubygems"
|
4
4
|
require "daemons"
|
5
|
+
require "eventmachine"
|
6
|
+
require "websocket-eventmachine-client"
|
5
7
|
require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
|
6
8
|
|
7
9
|
tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
|
8
10
|
system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
|
9
11
|
|
12
|
+
DEBUG = true
|
13
|
+
|
10
14
|
Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
|
11
15
|
Beetle.config.servers = "127.0.0.1:5672" # rabbitmq
|
12
16
|
|
13
17
|
# set Beetle log level to info, less noisy than debug
|
14
18
|
Beetle.config.logger.level = Logger::DEBUG
|
15
19
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
20
|
+
log_file = File.open(system_notification_log_file_path, "a+")
|
21
|
+
log_file.sync = true
|
22
|
+
|
23
|
+
@interrupted = false
|
24
|
+
def shutdown(ws)
|
25
|
+
@interrupted = true
|
26
|
+
ws.unbind
|
27
|
+
end
|
28
|
+
|
29
|
+
while !@interrupted
|
30
|
+
EventMachine.run do
|
31
|
+
ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://127.0.0.1:9650/notifications')
|
32
|
+
ws.onopen do
|
33
|
+
puts "established connection" if DEBUG
|
34
|
+
end
|
35
|
+
ws.onclose do
|
36
|
+
puts "server closed connection" if DEBUG && !@interrupted
|
37
|
+
EM.add_timer(1){ EM.stop_event_loop }
|
38
|
+
end
|
39
|
+
ws.onmessage do |text|
|
40
|
+
puts "writing message to #{system_notification_log_file_path}: #{text}"
|
41
|
+
log_file << text
|
26
42
|
end
|
43
|
+
puts "Started system notification logger"
|
44
|
+
trap("INT") { shutdown(ws) }
|
45
|
+
trap("TERM") { shutdown(ws) }
|
27
46
|
end
|
28
47
|
end
|
29
|
-
puts "Started system notification logger"
|
30
|
-
client.listen
|
31
48
|
end
|
@@ -36,11 +36,11 @@ module TestDaemons
|
|
36
36
|
def daemon_controller
|
37
37
|
@daemon_controller ||= DaemonController.new(
|
38
38
|
:identifier => "Redis configuration test client #{@name}",
|
39
|
-
:start_command => "
|
39
|
+
:start_command => "./beetle configuration_client -v -d --redis-master-file #{redis_master_file} --id #{@name} --pid-file #{pid_file} --log-file #{log_file}",
|
40
40
|
:ping_command => lambda{ true },
|
41
41
|
:pid_file => pid_file,
|
42
42
|
:log_file => log_file,
|
43
|
-
:start_timeout => 5
|
43
|
+
:start_timeout => 5,
|
44
44
|
)
|
45
45
|
end
|
46
46
|
|
@@ -53,7 +53,7 @@ module TestDaemons
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def log_file
|
56
|
-
"#{tmp_path}/
|
56
|
+
"#{tmp_path}/redis_configuration_client_num#{@daemon_id}.output"
|
57
57
|
end
|
58
58
|
|
59
59
|
def tmp_path
|
@@ -25,7 +25,7 @@ module TestDaemons
|
|
25
25
|
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
26
26
|
DaemonController.new(
|
27
27
|
:identifier => "Redis configuration test server",
|
28
|
-
:start_command => "
|
28
|
+
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file}",
|
29
29
|
:ping_command => lambda{ answers_text_requests? },
|
30
30
|
:pid_file => pid_file,
|
31
31
|
:log_file => log_file,
|
@@ -74,7 +74,7 @@ module TestDaemons
|
|
74
74
|
false
|
75
75
|
end
|
76
76
|
|
77
|
-
HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ?
|
77
|
+
HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9650 : 8080
|
78
78
|
|
79
79
|
def self.get_status(path, content_type)
|
80
80
|
uri = URI.parse("http://127.0.0.1:#{HTTP_SERVER_PORT}#{path}")
|
@@ -48,72 +48,6 @@ module Beetle
|
|
48
48
|
key =~ /^(msgid:[^:]*:[-0-9a-f]*):.*$/ && $1
|
49
49
|
end
|
50
50
|
|
51
|
-
# garbage collect using master and slave (and redis-cli)
|
52
|
-
def garbage_collect_keys_using_master_and_slave(now = Time.now.to_i)
|
53
|
-
logger.info "garbage collecting obsolete redis keys from #{@config.redis_servers}"
|
54
|
-
info = RedisServerInfo.new(@config, {})
|
55
|
-
info.refresh
|
56
|
-
unless connection = info.slaves.first
|
57
|
-
logger.warn "no slave available, falling back to master."
|
58
|
-
connection = redis
|
59
|
-
end
|
60
|
-
file = "#{@config.tmpdir}/beetle_redis_expire_keys_#{$$}.txt"
|
61
|
-
cmd = "redis-cli -h #{connection.host} -p #{connection.port} -n #{@config.redis_db} keys 'msgid:*:expires' > #{file}"
|
62
|
-
logger.info "retrieving expire keys: '#{cmd}'"
|
63
|
-
if system(cmd)
|
64
|
-
garbage_collect_keys_from_file(file, now)
|
65
|
-
else
|
66
|
-
logger.error "could not retrieve expire keys"
|
67
|
-
end
|
68
|
-
rescue => e
|
69
|
-
logger.error "#{e.class}(#{e})"
|
70
|
-
ensure
|
71
|
-
system("rm -f #{file}")
|
72
|
-
end
|
73
|
-
|
74
|
-
# garbage collect keys in Redis (always assume the worst!)
|
75
|
-
def garbage_collect_keys(now = Time.now.to_i)
|
76
|
-
keys = redis.keys("msgid:*:expires")
|
77
|
-
threshold = now - @config.gc_threshold
|
78
|
-
keys.each do |key|
|
79
|
-
gc_key(key, threshold)
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
# garbage collect keys from a given file
|
84
|
-
def garbage_collect_keys_from_file(file_name, now = Time.now.to_i)
|
85
|
-
threshold = now - @config.gc_threshold
|
86
|
-
expired = total = 0
|
87
|
-
File.foreach(file_name) do |line|
|
88
|
-
line.chomp!
|
89
|
-
if line =~ /^msgid:.*:expires$/
|
90
|
-
total += 1
|
91
|
-
expired += 1 if gc_key(line, threshold)
|
92
|
-
end
|
93
|
-
end
|
94
|
-
logger.info "expired #{expired} keys out of #{total}"
|
95
|
-
end
|
96
|
-
|
97
|
-
# garbage collect a single key if it's older than given threshold
|
98
|
-
def gc_key(key, threshold)
|
99
|
-
expires_at = redis.get key
|
100
|
-
if expires_at
|
101
|
-
if expires_at.to_i < threshold
|
102
|
-
msg_id = msg_id(key)
|
103
|
-
# puts "expiring: #{msg_id}"
|
104
|
-
redis.del(*keys(msg_id))
|
105
|
-
return true
|
106
|
-
else
|
107
|
-
# key has future expiry date
|
108
|
-
# puts "#{msg_id} expires on #{Time.at(expires_at.to_i)}"
|
109
|
-
return false
|
110
|
-
end
|
111
|
-
else
|
112
|
-
# key might have been deleted in the meantime
|
113
|
-
return false
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
51
|
# unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
|
118
52
|
def set(msg_id, suffix, value)
|
119
53
|
with_failover { redis.set(key(msg_id, suffix), value) }
|
@@ -26,10 +26,5 @@ module Beetle
|
|
26
26
|
config.redis_server
|
27
27
|
end
|
28
28
|
|
29
|
-
def verify_redis_master_file_string
|
30
|
-
if master_file =~ /^[0-9a-z.]+:[0-9]+$/
|
31
|
-
raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
|
32
|
-
end
|
33
|
-
end
|
34
29
|
end
|
35
30
|
end
|
data/lib/beetle/version.rb
CHANGED
@@ -111,52 +111,4 @@ module Beetle
|
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
|
-
class GarbageCollectionTest < Minitest::Test
|
115
|
-
def setup
|
116
|
-
@store = DeduplicationStore.new
|
117
|
-
Beetle.config.stubs(:gc_threshold).returns(10)
|
118
|
-
end
|
119
|
-
|
120
|
-
test "never tries to delete message keys when expire key does not exist" do
|
121
|
-
key = "foo"
|
122
|
-
@store.redis.del key
|
123
|
-
@store.redis.expects(:del).never
|
124
|
-
assert !@store.gc_key(key, 0)
|
125
|
-
end
|
126
|
-
|
127
|
-
test "rescues exeptions and logs an error" do
|
128
|
-
RedisServerInfo.expects(:new).raises("foo")
|
129
|
-
assert_nothing_raised { @store.garbage_collect_keys_using_master_and_slave }
|
130
|
-
end
|
131
|
-
|
132
|
-
test "logs an error when system command fails" do
|
133
|
-
@store.stubs(:system).returns(false)
|
134
|
-
@store.logger.expects(:error)
|
135
|
-
@store.garbage_collect_keys_using_master_and_slave
|
136
|
-
end
|
137
|
-
|
138
|
-
test "garbage collects a key when it has expired" do
|
139
|
-
key = "foo"
|
140
|
-
t = Time.now.to_i
|
141
|
-
@store.redis.set(key, t)
|
142
|
-
@store.redis.expects(:del)
|
143
|
-
assert @store.gc_key(key, t+1)
|
144
|
-
end
|
145
|
-
|
146
|
-
test "does not garbage collect a key when it has not expired" do
|
147
|
-
key = "foo"
|
148
|
-
t = Time.now.to_i
|
149
|
-
@store.redis.set(key, t)
|
150
|
-
@store.redis.expects(:del).never
|
151
|
-
assert !@store.gc_key(key, t)
|
152
|
-
end
|
153
|
-
|
154
|
-
test "correctly sets threshold for garbage collection" do
|
155
|
-
t = Time.now.to_i
|
156
|
-
@store.redis.expects(:keys).returns(["foo"])
|
157
|
-
@store.expects(:gc_key).with("foo", t-10)
|
158
|
-
@store.garbage_collect_keys
|
159
|
-
end
|
160
|
-
|
161
|
-
end
|
162
114
|
end
|
data/test/beetle/message_test.rb
CHANGED
@@ -127,36 +127,6 @@ module Beetle
|
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
|
-
test "should be able to garbage collect expired keys" do
|
131
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
132
|
-
header = header_with_params({:ttl => 0})
|
133
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
134
|
-
assert !message.key_exists?
|
135
|
-
assert message.key_exists?
|
136
|
-
@store.redis.expects(:del).with(*@store.keys(message.msg_id))
|
137
|
-
@store.garbage_collect_keys(Time.now.to_i+11)
|
138
|
-
end
|
139
|
-
|
140
|
-
test "should be able to garbage collect expired keys using master and slave" do
|
141
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
142
|
-
header = header_with_params({:ttl => 0})
|
143
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
144
|
-
assert !message.key_exists?
|
145
|
-
assert message.key_exists?
|
146
|
-
@store.redis.expects(:del).with(*@store.keys(message.msg_id))
|
147
|
-
@store.garbage_collect_keys_using_master_and_slave(Time.now.to_i+11)
|
148
|
-
end
|
149
|
-
|
150
|
-
test "should not garbage collect not yet expired keys" do
|
151
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
152
|
-
header = header_with_params({:ttl => 0})
|
153
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
154
|
-
assert !message.key_exists?
|
155
|
-
assert message.key_exists?
|
156
|
-
@store.redis.expects(:del).never
|
157
|
-
@store.garbage_collect_keys(Time.now.to_i)
|
158
|
-
end
|
159
|
-
|
160
130
|
test "successful processing of a non redundant message should delete all keys from the database" do
|
161
131
|
header = header_with_params({})
|
162
132
|
header.expects(:ack)
|