beetle 1.0.3 → 2.0.0rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +15 -7
- data/REDIS_AUTO_FAILOVER.rdoc +8 -16
- data/RELEASE_NOTES.rdoc +9 -0
- data/beetle.gemspec +22 -10
- data/features/support/system_notification_logger +29 -12
- data/features/support/test_daemons/redis.rb +1 -1
- data/features/support/test_daemons/redis_configuration_client.rb +3 -3
- data/features/support/test_daemons/redis_configuration_server.rb +2 -2
- data/lib/beetle/deduplication_store.rb +0 -66
- data/lib/beetle/message.rb +1 -1
- data/lib/beetle/publisher.rb +2 -1
- data/lib/beetle/redis_master_file.rb +0 -5
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +0 -48
- data/test/beetle/message_test.rb +1 -31
- data/test/beetle/publisher_test.rb +2 -1
- metadata +165 -26
- data/bin/beetle +0 -9
- data/lib/beetle/commands.rb +0 -35
- data/lib/beetle/commands/configuration_client.rb +0 -98
- data/lib/beetle/commands/configuration_server.rb +0 -98
- data/lib/beetle/commands/garbage_collect_deduplication_store.rb +0 -52
- data/lib/beetle/redis_configuration_client.rb +0 -157
- data/lib/beetle/redis_configuration_http_server.rb +0 -152
- data/lib/beetle/redis_configuration_server.rb +0 -438
- data/lib/beetle/redis_server_info.rb +0 -66
- data/script/docker-run-beetle-tests- +0 -5
- data/test/beetle/redis_configuration_client_test.rb +0 -118
- data/test/beetle/redis_configuration_server_test.rb +0 -381
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5648bfe9f8b8f76d05d04b360df2e160c7f9ab6b
|
4
|
+
data.tar.gz: 5fdb6ec4eea81aa6e7d79bbbdd9ed820711bdb2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e5e5f2ba5cb31942fbf01584cae8b448cec7db14e260050e74fb5765c9d29a23c9f7d1cd8e93cf2be04ea74d46739e57c45149fe9c4557e3fce77936814d6ed5
|
7
|
+
data.tar.gz: 28b1e12e6fc4b9858c6184ff93dbc876cc51ced13cac9f31664547b07fbf9dfa5557c83cc7d42a609cda20bee8f00b85d6a9520aa28290ab7172a141eaa7a93f
|
data/README.rdoc
CHANGED
@@ -29,10 +29,10 @@ More information can be found on the {project website}[http://xing.github.com/be
|
|
29
29
|
|
30
30
|
# configure exchanges, queues, bindings, messages and handlers
|
31
31
|
|
32
|
-
b.configure do
|
33
|
-
|
34
|
-
|
35
|
-
|
32
|
+
b.configure do
|
33
|
+
queue :test
|
34
|
+
message :test
|
35
|
+
handler(:test) { |message| puts message.data }
|
36
36
|
end
|
37
37
|
|
38
38
|
=== Publishing
|
@@ -72,6 +72,13 @@ You also need a Redis instance running. The default configuration of Redis will
|
|
72
72
|
|
73
73
|
redis-server
|
74
74
|
|
75
|
+
If you want to run the integration tests you need GO installed and you
|
76
|
+
will need to build the beetle binary. We provide a Makefile for this
|
77
|
+
purpose, so simply running
|
78
|
+
|
79
|
+
make
|
80
|
+
|
81
|
+
should suffice.
|
75
82
|
|
76
83
|
== Gem Dependencies
|
77
84
|
|
@@ -127,8 +134,9 @@ Update RELEASE_NOTES.rdoc!
|
|
127
134
|
We use {semantic versioning}[http://semver.org/] and create a git tag
|
128
135
|
for each release.
|
129
136
|
|
130
|
-
Edit `lib/beetle/version.rb`
|
131
|
-
|
137
|
+
Edit `lib/beetle/version.rb` and
|
138
|
+
`go/src/github.com/xing/beetle/version.go` to set the new version
|
139
|
+
number (`Major.Minor.Patch`).
|
132
140
|
|
133
141
|
In short (see {semver.org}[http://semver.org] for details):
|
134
142
|
|
@@ -141,7 +149,7 @@ In short (see {semver.org}[http://semver.org] for details):
|
|
141
149
|
are introduced.
|
142
150
|
|
143
151
|
Then use `rake release` which will create the git tag and upload the
|
144
|
-
gem to
|
152
|
+
gem to github.com:
|
145
153
|
|
146
154
|
$ bundle exec rake release
|
147
155
|
|
data/REDIS_AUTO_FAILOVER.rdoc
CHANGED
@@ -35,9 +35,9 @@ On every worker server runs another daemon, the Redis Configuration Client
|
|
35
35
|
|
36
36
|
If the RCS finds another potential Redis Master, it sends out a message to see
|
37
37
|
if all known RCCs are still available (once again to eliminate the risk of a
|
38
|
-
partitioned network) and if they agree to the master switch.
|
38
|
+
partitioned network) and if they agree to the master switch.
|
39
39
|
|
40
|
-
If all RCCs have answered to that message, the RCS sends out a message which
|
40
|
+
If all RCCs have answered to that message, the RCS sends out a message which
|
41
41
|
tells the RCCs to invalidate the current master.
|
42
42
|
|
43
43
|
This happens by deleting the contents of a special file which is used
|
@@ -53,7 +53,7 @@ to the RCCs, which then write that value into their redis master file.
|
|
53
53
|
|
54
54
|
Additionally, the RCS sends reconfigure messages with the current Redis master
|
55
55
|
periodically, to allow new RCCs to pick up the current master. Plus it turns
|
56
|
-
all other redis servers into slaves of the current master.
|
56
|
+
all other redis servers into slaves of the current master.
|
57
57
|
|
58
58
|
=== Prerequisites
|
59
59
|
|
@@ -93,15 +93,11 @@ All command line options can also be given as a yaml configuration file via the
|
|
93
93
|
|
94
94
|
Start the Redis Configuration Server:
|
95
95
|
|
96
|
-
beetle configuration_server
|
97
|
-
|
98
|
-
Get help for starting/stopping the server:
|
99
|
-
|
100
|
-
beetle configuration_server -h
|
96
|
+
beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
|
101
97
|
|
102
98
|
Get help for server options:
|
103
99
|
|
104
|
-
beetle configuration_server
|
100
|
+
beetle configuration_server -h
|
105
101
|
|
106
102
|
=== On every worker server
|
107
103
|
|
@@ -109,16 +105,12 @@ Start the Redis Configuration Client:
|
|
109
105
|
|
110
106
|
On first worker server:
|
111
107
|
|
112
|
-
beetle configuration_client
|
108
|
+
beetle configuration_client --client-id rcc-1
|
113
109
|
|
114
110
|
On second worker server:
|
115
111
|
|
116
|
-
beetle configuration_client
|
117
|
-
|
118
|
-
Get help for starting/stopping the client:
|
119
|
-
|
120
|
-
beetle configuration_client -h
|
112
|
+
beetle configuration_client --client-id rcc-2
|
121
113
|
|
122
114
|
Get help for client options:
|
123
115
|
|
124
|
-
beetle configuration_client
|
116
|
+
beetle configuration_client -h
|
data/RELEASE_NOTES.rdoc
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
= Release Notes
|
2
2
|
|
3
|
+
== Version 2.0.0rc1
|
4
|
+
|
5
|
+
* beetle command has been rewritten in GO
|
6
|
+
* garbage collecting redis keys now uses the redis SCAN command
|
7
|
+
|
8
|
+
== Version 1.0.4
|
9
|
+
|
10
|
+
* use amqp protocol version 0.9 by default for publishers
|
11
|
+
|
3
12
|
== Version 1.0.3
|
4
13
|
|
5
14
|
* fixed that publisher did not allow specifying message properties
|
data/beetle.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.description = "A highly available, reliable messaging infrastructure"
|
13
13
|
s.summary = "High Availability AMQP Messaging with Redundant Queues"
|
14
14
|
s.email = "opensource@xing.com"
|
15
|
-
s.executables = [
|
15
|
+
s.executables = []
|
16
16
|
s.extra_rdoc_files = Dir['**/*.rdoc'] + %w(MIT-LICENSE)
|
17
17
|
s.files = Dir['{examples,lib}/**/*.rb'] + Dir['{features,script}/**/*'] + %w(beetle.gemspec Rakefile)
|
18
18
|
s.homepage = "http://xing.github.com/beetle/"
|
@@ -21,13 +21,25 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.test_files = Dir['test/**/*.rb']
|
22
22
|
|
23
23
|
s.specification_version = 3
|
24
|
-
s.add_runtime_dependency
|
25
|
-
s.add_runtime_dependency
|
26
|
-
s.add_runtime_dependency
|
27
|
-
s.add_runtime_dependency
|
28
|
-
s.add_runtime_dependency
|
29
|
-
s.add_runtime_dependency
|
30
|
-
s.add_runtime_dependency
|
31
|
-
|
32
|
-
s.
|
24
|
+
s.add_runtime_dependency "uuid4r", ">= 0.1.2"
|
25
|
+
s.add_runtime_dependency "bunny", "~> 0.7.10"
|
26
|
+
s.add_runtime_dependency "redis", ">= 2.2.2"
|
27
|
+
s.add_runtime_dependency "hiredis", ">= 0.4.5"
|
28
|
+
s.add_runtime_dependency "amq-protocol", "= 2.0.1"
|
29
|
+
s.add_runtime_dependency "amqp", "= 1.6.0"
|
30
|
+
s.add_runtime_dependency "activesupport", ">= 2.3.4"
|
31
|
+
|
32
|
+
s.add_development_dependency "activerecord", "~> 5.0"
|
33
|
+
s.add_development_dependency "cucumber", "~> 2.4.0"
|
34
|
+
s.add_development_dependency "daemon_controller", "~> 1.2.0"
|
35
|
+
s.add_development_dependency "daemons", ">= 1.2.0"
|
36
|
+
s.add_development_dependency "i18n"
|
37
|
+
s.add_development_dependency "minitest", "~> 5.1"
|
38
|
+
s.add_development_dependency "mocha", "~> 1.1.0"
|
39
|
+
s.add_development_dependency "mysql2", "~> 0.4.4"
|
40
|
+
s.add_development_dependency "rake", "~> 11.2"
|
41
|
+
s.add_development_dependency "rdoc", "~> 4.0"
|
42
|
+
s.add_development_dependency "simplecov", "~> 0.12.0"
|
43
|
+
s.add_development_dependency "webmock", "~> 1.21.0"
|
44
|
+
s.add_development_dependency "websocket-eventmachine-client"
|
33
45
|
end
|
@@ -2,30 +2,47 @@
|
|
2
2
|
|
3
3
|
require "rubygems"
|
4
4
|
require "daemons"
|
5
|
+
require "eventmachine"
|
6
|
+
require "websocket-eventmachine-client"
|
5
7
|
require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
|
6
8
|
|
7
9
|
tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
|
8
10
|
system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
|
9
11
|
|
12
|
+
DEBUG = true
|
13
|
+
|
10
14
|
Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
|
11
15
|
Beetle.config.servers = "127.0.0.1:5672" # rabbitmq
|
12
16
|
|
13
17
|
# set Beetle log level to info, less noisy than debug
|
14
18
|
Beetle.config.logger.level = Logger::DEBUG
|
15
19
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
20
|
+
log_file = File.open(system_notification_log_file_path, "a+")
|
21
|
+
log_file.sync = true
|
22
|
+
|
23
|
+
@interrupted = false
|
24
|
+
def shutdown(ws)
|
25
|
+
@interrupted = true
|
26
|
+
ws.unbind
|
27
|
+
end
|
28
|
+
|
29
|
+
while !@interrupted
|
30
|
+
EventMachine.run do
|
31
|
+
ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://127.0.0.1:9650/notifications')
|
32
|
+
ws.onopen do
|
33
|
+
puts "established connection" if DEBUG
|
34
|
+
end
|
35
|
+
ws.onclose do
|
36
|
+
puts "server closed connection" if DEBUG && !@interrupted
|
37
|
+
EM.add_timer(1){ EM.stop_event_loop }
|
38
|
+
end
|
39
|
+
ws.onmessage do |text|
|
40
|
+
puts "writing message to #{system_notification_log_file_path}: #{text}"
|
41
|
+
log_file << text
|
26
42
|
end
|
43
|
+
puts "Started system notification logger"
|
44
|
+
trap("INT") { shutdown(ws) }
|
45
|
+
trap("TERM") { shutdown(ws) }
|
27
46
|
end
|
28
47
|
end
|
29
|
-
puts "Started system notification logger"
|
30
|
-
client.listen
|
31
48
|
end
|
@@ -36,11 +36,11 @@ module TestDaemons
|
|
36
36
|
def daemon_controller
|
37
37
|
@daemon_controller ||= DaemonController.new(
|
38
38
|
:identifier => "Redis configuration test client #{@name}",
|
39
|
-
:start_command => "
|
39
|
+
:start_command => "./beetle configuration_client -v -d --redis-master-file #{redis_master_file} --id #{@name} --pid-file #{pid_file} --log-file #{log_file}",
|
40
40
|
:ping_command => lambda{ true },
|
41
41
|
:pid_file => pid_file,
|
42
42
|
:log_file => log_file,
|
43
|
-
:start_timeout => 5
|
43
|
+
:start_timeout => 5,
|
44
44
|
)
|
45
45
|
end
|
46
46
|
|
@@ -53,7 +53,7 @@ module TestDaemons
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def log_file
|
56
|
-
"#{tmp_path}/
|
56
|
+
"#{tmp_path}/redis_configuration_client_num#{@daemon_id}.output"
|
57
57
|
end
|
58
58
|
|
59
59
|
def tmp_path
|
@@ -25,7 +25,7 @@ module TestDaemons
|
|
25
25
|
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
26
26
|
DaemonController.new(
|
27
27
|
:identifier => "Redis configuration test server",
|
28
|
-
:start_command => "
|
28
|
+
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file}",
|
29
29
|
:ping_command => lambda{ answers_text_requests? },
|
30
30
|
:pid_file => pid_file,
|
31
31
|
:log_file => log_file,
|
@@ -74,7 +74,7 @@ module TestDaemons
|
|
74
74
|
false
|
75
75
|
end
|
76
76
|
|
77
|
-
HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ?
|
77
|
+
HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9650 : 8080
|
78
78
|
|
79
79
|
def self.get_status(path, content_type)
|
80
80
|
uri = URI.parse("http://127.0.0.1:#{HTTP_SERVER_PORT}#{path}")
|
@@ -48,72 +48,6 @@ module Beetle
|
|
48
48
|
key =~ /^(msgid:[^:]*:[-0-9a-f]*):.*$/ && $1
|
49
49
|
end
|
50
50
|
|
51
|
-
# garbage collect using master and slave (and redis-cli)
|
52
|
-
def garbage_collect_keys_using_master_and_slave(now = Time.now.to_i)
|
53
|
-
logger.info "garbage collecting obsolete redis keys from #{@config.redis_servers}"
|
54
|
-
info = RedisServerInfo.new(@config, {})
|
55
|
-
info.refresh
|
56
|
-
unless connection = info.slaves.first
|
57
|
-
logger.warn "no slave available, falling back to master."
|
58
|
-
connection = redis
|
59
|
-
end
|
60
|
-
file = "#{@config.tmpdir}/beetle_redis_expire_keys_#{$$}.txt"
|
61
|
-
cmd = "redis-cli -h #{connection.host} -p #{connection.port} -n #{@config.redis_db} keys 'msgid:*:expires' > #{file}"
|
62
|
-
logger.info "retrieving expire keys: '#{cmd}'"
|
63
|
-
if system(cmd)
|
64
|
-
garbage_collect_keys_from_file(file, now)
|
65
|
-
else
|
66
|
-
logger.error "could not retrieve expire keys"
|
67
|
-
end
|
68
|
-
rescue => e
|
69
|
-
logger.error "#{e.class}(#{e})"
|
70
|
-
ensure
|
71
|
-
system("rm -f #{file}")
|
72
|
-
end
|
73
|
-
|
74
|
-
# garbage collect keys in Redis (always assume the worst!)
|
75
|
-
def garbage_collect_keys(now = Time.now.to_i)
|
76
|
-
keys = redis.keys("msgid:*:expires")
|
77
|
-
threshold = now - @config.gc_threshold
|
78
|
-
keys.each do |key|
|
79
|
-
gc_key(key, threshold)
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
# garbage collect keys from a given file
|
84
|
-
def garbage_collect_keys_from_file(file_name, now = Time.now.to_i)
|
85
|
-
threshold = now - @config.gc_threshold
|
86
|
-
expired = total = 0
|
87
|
-
File.foreach(file_name) do |line|
|
88
|
-
line.chomp!
|
89
|
-
if line =~ /^msgid:.*:expires$/
|
90
|
-
total += 1
|
91
|
-
expired += 1 if gc_key(line, threshold)
|
92
|
-
end
|
93
|
-
end
|
94
|
-
logger.info "expired #{expired} keys out of #{total}"
|
95
|
-
end
|
96
|
-
|
97
|
-
# garbage collect a single key if it's older than given threshold
|
98
|
-
def gc_key(key, threshold)
|
99
|
-
expires_at = redis.get key
|
100
|
-
if expires_at
|
101
|
-
if expires_at.to_i < threshold
|
102
|
-
msg_id = msg_id(key)
|
103
|
-
# puts "expiring: #{msg_id}"
|
104
|
-
redis.del(*keys(msg_id))
|
105
|
-
return true
|
106
|
-
else
|
107
|
-
# key has future expiry date
|
108
|
-
# puts "#{msg_id} expires on #{Time.at(expires_at.to_i)}"
|
109
|
-
return false
|
110
|
-
end
|
111
|
-
else
|
112
|
-
# key might have been deleted in the meantime
|
113
|
-
return false
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
51
|
# unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
|
118
52
|
def set(msg_id, suffix, value)
|
119
53
|
with_failover { redis.set(key(msg_id, suffix), value) }
|
data/lib/beetle/message.rb
CHANGED
data/lib/beetle/publisher.rb
CHANGED
@@ -166,7 +166,8 @@ module Beetle
|
|
166
166
|
:pass => @client.config.password,
|
167
167
|
:vhost => @client.config.vhost,
|
168
168
|
:frame_max => @client.config.frame_max,
|
169
|
-
:socket_timeout => @client.config.publishing_timeout
|
169
|
+
:socket_timeout => @client.config.publishing_timeout,
|
170
|
+
:spec => '09')
|
170
171
|
b.start
|
171
172
|
b
|
172
173
|
end
|
@@ -26,10 +26,5 @@ module Beetle
|
|
26
26
|
config.redis_server
|
27
27
|
end
|
28
28
|
|
29
|
-
def verify_redis_master_file_string
|
30
|
-
if master_file =~ /^[0-9a-z.]+:[0-9]+$/
|
31
|
-
raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
|
32
|
-
end
|
33
|
-
end
|
34
29
|
end
|
35
30
|
end
|
data/lib/beetle/version.rb
CHANGED
@@ -111,52 +111,4 @@ module Beetle
|
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
|
-
class GarbageCollectionTest < Minitest::Test
|
115
|
-
def setup
|
116
|
-
@store = DeduplicationStore.new
|
117
|
-
Beetle.config.stubs(:gc_threshold).returns(10)
|
118
|
-
end
|
119
|
-
|
120
|
-
test "never tries to delete message keys when expire key does not exist" do
|
121
|
-
key = "foo"
|
122
|
-
@store.redis.del key
|
123
|
-
@store.redis.expects(:del).never
|
124
|
-
assert !@store.gc_key(key, 0)
|
125
|
-
end
|
126
|
-
|
127
|
-
test "rescues exeptions and logs an error" do
|
128
|
-
RedisServerInfo.expects(:new).raises("foo")
|
129
|
-
assert_nothing_raised { @store.garbage_collect_keys_using_master_and_slave }
|
130
|
-
end
|
131
|
-
|
132
|
-
test "logs an error when system command fails" do
|
133
|
-
@store.stubs(:system).returns(false)
|
134
|
-
@store.logger.expects(:error)
|
135
|
-
@store.garbage_collect_keys_using_master_and_slave
|
136
|
-
end
|
137
|
-
|
138
|
-
test "garbage collects a key when it has expired" do
|
139
|
-
key = "foo"
|
140
|
-
t = Time.now.to_i
|
141
|
-
@store.redis.set(key, t)
|
142
|
-
@store.redis.expects(:del)
|
143
|
-
assert @store.gc_key(key, t+1)
|
144
|
-
end
|
145
|
-
|
146
|
-
test "does not garbage collect a key when it has not expired" do
|
147
|
-
key = "foo"
|
148
|
-
t = Time.now.to_i
|
149
|
-
@store.redis.set(key, t)
|
150
|
-
@store.redis.expects(:del).never
|
151
|
-
assert !@store.gc_key(key, t)
|
152
|
-
end
|
153
|
-
|
154
|
-
test "correctly sets threshold for garbage collection" do
|
155
|
-
t = Time.now.to_i
|
156
|
-
@store.redis.expects(:keys).returns(["foo"])
|
157
|
-
@store.expects(:gc_key).with("foo", t-10)
|
158
|
-
@store.garbage_collect_keys
|
159
|
-
end
|
160
|
-
|
161
|
-
end
|
162
114
|
end
|