beetle 2.2.3 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.rdoc +1 -1
- data/RELEASE_NOTES.rdoc +21 -1
- data/beetle.gemspec +5 -2
- data/examples/attempts.rb +1 -0
- data/features/redis_auto_failover.feature +46 -0
- data/features/step_definitions/redis_auto_failover_steps.rb +27 -12
- data/features/support/beetle_handler +9 -2
- data/features/support/env.rb +6 -2
- data/features/support/test_daemons/redis_configuration_server.rb +4 -4
- data/lib/beetle.rb +0 -1
- data/lib/beetle/configuration.rb +13 -0
- data/lib/beetle/dead_lettering.rb +36 -26
- data/lib/beetle/deduplication_store.rb +19 -2
- data/lib/beetle/message.rb +10 -2
- data/lib/beetle/publisher.rb +12 -10
- data/lib/beetle/version.rb +1 -1
- data/script/console +4 -4
- data/test/beetle/dead_lettering_test.rb +36 -15
- data/test/beetle/deduplication_store_test.rb +13 -0
- data/test/beetle/publisher_test.rb +1 -0
- metadata +20 -23
- data/features/#redis_auto_failover.feature# +0 -165
- data/lib/beetle/redis_master_file.rb +0 -30
- data/test/beetle/redis_master_file_test.rb +0 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c8a5f1b489f657fd5e286f6461b13ee314b6b4abe3dae6452bb53ca81ce23255
|
4
|
+
data.tar.gz: 183a0d96adf42fe4648f946203527edd1b065b7c961b62b825e6f5c6483edf7e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3eaa2e3b642b8c3dd5d5062a4327d5499e87c3c02d27493b33f9f42fa1bb2964e2f6b5af386c88525b2343e075ba61e42fe9a75ce459d2a98ef949e418ffe7e1
|
7
|
+
data.tar.gz: cf80066e492795627ef460eec5c05a222a5af569d87e62084ec865939e715a6f4c6ca902f618d9039f7d650c9208b694a30e28dd81bd23571d71b1a1ff7ea287
|
data/README.rdoc
CHANGED
@@ -111,7 +111,7 @@ For tests, you'll need
|
|
111
111
|
|
112
112
|
You can find out more about our work on our {dev blog}[http://devblog.xing.com].
|
113
113
|
|
114
|
-
Copyright (c) 2010-
|
114
|
+
Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
|
115
115
|
|
116
116
|
Released under the MIT license. For full details see MIT-LICENSE included in this
|
117
117
|
distribution.
|
data/RELEASE_NOTES.rdoc
CHANGED
@@ -1,7 +1,27 @@
|
|
1
1
|
= Release Notes
|
2
2
|
|
3
|
+
== Version 2.3.0
|
4
|
+
* redis failover: support multiple redis failover instances. This has
|
5
|
+
required a change in the redis master file format. The master file
|
6
|
+
content is now either the old format (host:port) for systems using a
|
7
|
+
single redis failover system or a mapping from system names to
|
8
|
+
host:port combinations. For example,
|
9
|
+
"system1/master1:6379\nsystem2/master" specifies to systems with
|
10
|
+
their corresponding redis masters. Beetle client uses the configured
|
11
|
+
system name to tind the master it should use.
|
12
|
+
* support lazy queues: setting :lazy_quques_enabled on the beetle
|
13
|
+
cofiguration will enable queue_mode: "lazy" for all queues of
|
14
|
+
declared on the beetle client.
|
15
|
+
* improved calculation of channel close an connection disconnect
|
16
|
+
timeouts for beetle publisher to avoid warnings in RabbitMQ logs.
|
17
|
+
* use SecureRandom.uuid instead of UUID4R::uuid(4) if UUID4R cannot
|
18
|
+
be loaded.
|
19
|
+
|
20
|
+
== Version 2.2.4
|
21
|
+
* redis failover: prevent starting a new master switch while one is running
|
22
|
+
|
3
23
|
== Version 2.2.3
|
4
|
-
* redis failover server logs errors when redis oparations fail
|
24
|
+
* redis failover: server logs errors when redis oparations fail
|
5
25
|
|
6
26
|
== Version 2.2.2
|
7
27
|
|
data/beetle.gemspec
CHANGED
@@ -18,16 +18,19 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.rdoc_options = ["--charset=UTF-8"]
|
19
19
|
s.require_paths = ["lib"]
|
20
20
|
s.test_files = Dir['test/**/*.rb']
|
21
|
+
s.metadata = {
|
22
|
+
"changelog_uri" => "https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc"
|
23
|
+
}
|
21
24
|
|
22
25
|
s.specification_version = 3
|
23
|
-
s.add_runtime_dependency "
|
24
|
-
s.add_runtime_dependency "bunny", "~> 0.7.10"
|
26
|
+
s.add_runtime_dependency "bunny", "~> 0.7.12"
|
25
27
|
s.add_runtime_dependency "redis", ">= 2.2.2"
|
26
28
|
s.add_runtime_dependency "hiredis", ">= 0.4.5"
|
27
29
|
s.add_runtime_dependency "amq-protocol", "= 2.0.1"
|
28
30
|
s.add_runtime_dependency "amqp", "= 1.6.0"
|
29
31
|
s.add_runtime_dependency "activesupport", ">= 2.3.4"
|
30
32
|
|
33
|
+
s.add_development_dependency "uuid4r", ">= 0.1.2"
|
31
34
|
s.add_development_dependency "activerecord", "~> 5.0"
|
32
35
|
s.add_development_dependency "cucumber", "~> 2.4.0"
|
33
36
|
s.add_development_dependency "daemon_controller", "~> 1.2.0"
|
data/examples/attempts.rb
CHANGED
@@ -16,6 +16,7 @@ Beetle.config.logger.level = Logger::INFO
|
|
16
16
|
# setup client
|
17
17
|
$client = Beetle::Client.new
|
18
18
|
$client.config.dead_lettering_enabled = true
|
19
|
+
$client.config.lazy_queues_enabled = true
|
19
20
|
$client.configure(:key => "my.test.message") do
|
20
21
|
message(:test)
|
21
22
|
queue(:test)
|
@@ -23,6 +23,52 @@ Feature: Redis auto failover
|
|
23
23
|
Given a redis server "redis-1" exists as master
|
24
24
|
Then the role of redis server "redis-1" should be "slave"
|
25
25
|
|
26
|
+
Scenario: Successful single redis master switch with multiple failover sets
|
27
|
+
Given a redis server "redis-3" exists as master
|
28
|
+
And a redis server "redis-4" exists as slave of "redis-3"
|
29
|
+
Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
|
30
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
31
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
32
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
33
|
+
And redis server "redis-1" is down
|
34
|
+
And the retry timeout for the redis master check is reached
|
35
|
+
Then a system notification for "redis-1" not being available should be sent
|
36
|
+
And the role of redis server "redis-2" should be "master"
|
37
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
38
|
+
And the redis master file of the redis configuration server should contain "redis-3"
|
39
|
+
And the redis master of "rc-client-1" in system "a" should be "redis-2"
|
40
|
+
And the redis master of "rc-client-2" in system "a" should be "redis-2"
|
41
|
+
And the redis master of "rc-client-1" in system "b" should be "redis-3"
|
42
|
+
And the redis master of "rc-client-2" in system "b" should be "redis-3"
|
43
|
+
And the redis master of the beetle handler should be "redis-2"
|
44
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
45
|
+
Given a redis server "redis-1" exists as master
|
46
|
+
Then the role of redis server "redis-1" should be "slave"
|
47
|
+
|
48
|
+
Scenario: Successful double redis master switch with multiple failover sets
|
49
|
+
Given a redis server "redis-3" exists as master
|
50
|
+
And a redis server "redis-4" exists as slave of "redis-3"
|
51
|
+
Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
|
52
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
53
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
54
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
55
|
+
And redis server "redis-1" is down
|
56
|
+
And redis server "redis-3" is down
|
57
|
+
And the retry timeout for the redis master check is reached
|
58
|
+
And the role of redis server "redis-2" should be "master"
|
59
|
+
And the role of redis server "redis-4" should be "master"
|
60
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
61
|
+
And the redis master file of the redis configuration server should contain "redis-4"
|
62
|
+
And the redis master of "rc-client-1" in system "a" should be "redis-2"
|
63
|
+
And the redis master of "rc-client-2" in system "a" should be "redis-2"
|
64
|
+
And the redis master of "rc-client-1" in system "b" should be "redis-4"
|
65
|
+
And the redis master of "rc-client-2" in system "b" should be "redis-4"
|
66
|
+
And the redis master of the beetle handler should be "redis-2"
|
67
|
+
Given a redis server "redis-1" exists as master
|
68
|
+
Then the role of redis server "redis-1" should be "slave"
|
69
|
+
Given a redis server "redis-3" exists as master
|
70
|
+
Then the role of redis server "redis-3" should be "slave"
|
71
|
+
|
26
72
|
Scenario: Redis master only temporarily down (no switch necessary)
|
27
73
|
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
28
74
|
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
@@ -22,13 +22,22 @@ Given /^redis server "([^\"]*)" is slave of "([^\"]*)"$/ do |redis_name, redis_m
|
|
22
22
|
end
|
23
23
|
|
24
24
|
Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" (?:and confidence level "([^\"]*)" )?exists$/ do |redis_names, redis_configuration_client_names, confidence_level|
|
25
|
-
redis_servers = redis_names.split("
|
25
|
+
redis_servers = redis_names.split(";").map do |system_spec|
|
26
|
+
if system_spec.include?("/")
|
27
|
+
system_name, servers = system_spec.split("/", 2)
|
28
|
+
else
|
29
|
+
system_name, servers = nil, system_spec
|
30
|
+
end
|
31
|
+
servers = servers.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
|
32
|
+
system_name.nil? ? servers : "#{system_name}/#{servers}"
|
33
|
+
end.join(";")
|
26
34
|
TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names, (confidence_level || 100).to_i)
|
27
35
|
end
|
28
36
|
|
29
37
|
Given /^a redis configuration client "([^\"]*)" using redis servers "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_names|
|
30
|
-
|
31
|
-
|
38
|
+
redis_names.split(";").each do |system_spec|
|
39
|
+
servers = system_spec.sub(/^.*\//, '')
|
40
|
+
servers.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }
|
32
41
|
end
|
33
42
|
TestDaemons::RedisConfigurationClient[redis_configuration_client_name].start
|
34
43
|
end
|
@@ -73,7 +82,7 @@ end
|
|
73
82
|
Given /^an old redis master file for "([^\"]*)" with master "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_name|
|
74
83
|
master_file = redis_master_file(redis_configuration_client_name)
|
75
84
|
File.open(master_file, 'w') do |f|
|
76
|
-
f.puts TestDaemons::Redis[redis_name].ip_with_port
|
85
|
+
f.puts "system/#{TestDaemons::Redis[redis_name].ip_with_port}"
|
77
86
|
end
|
78
87
|
end
|
79
88
|
|
@@ -87,22 +96,28 @@ Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redi
|
|
87
96
|
assert expected_role, "#{redis_name} is not a #{role}"
|
88
97
|
end
|
89
98
|
|
90
|
-
Then /^the redis master of "([^\"]*)" should be "([^\"]*)"$/ do |redis_configuration_client_name, redis_name|
|
99
|
+
Then /^the redis master of "([^\"]*)" (?:in system "([^"]*)" )?should be "([^\"]*)"$/ do |redis_configuration_client_name, system_name, redis_name|
|
100
|
+
system_name ||= "system"
|
91
101
|
master_file = redis_master_file(redis_configuration_client_name)
|
92
102
|
master = false
|
93
|
-
server_info =
|
103
|
+
server_info = ''
|
94
104
|
10.times do
|
105
|
+
server_name = TestDaemons::Redis[redis_name].ip_with_port
|
95
106
|
server_info = File.read(master_file).chomp if File.exist?(master_file)
|
96
|
-
|
107
|
+
if server_info.include?("/")
|
108
|
+
master = true and break if server_info =~ /#{system_name}\/#{server_name}/m
|
109
|
+
else
|
110
|
+
master = true and break if server_info == server_name
|
111
|
+
end
|
97
112
|
sleep 1
|
98
113
|
end
|
99
114
|
assert master, "#{redis_name} is not master of #{redis_configuration_client_name}, master file content: #{server_info.inspect}"
|
100
115
|
end
|
101
116
|
|
102
|
-
Then /^the redis master file of the redis configuration server should contain "([^"]*)"$/ do |redis_name|
|
117
|
+
Then /^the redis master file of the redis configuration server should contain "([^"]*)"$/ do |redis_name|
|
103
118
|
master_file = TestDaemons::RedisConfigurationServer.redis_master_file
|
104
119
|
file_contents = File.read(master_file).chomp
|
105
|
-
|
120
|
+
assert_match /#{TestDaemons::Redis[redis_name].ip_with_port}/, file_contents
|
106
121
|
end
|
107
122
|
|
108
123
|
Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configuration_client_name|
|
@@ -111,7 +126,7 @@ Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configurat
|
|
111
126
|
server_info = nil
|
112
127
|
10.times do
|
113
128
|
server_info = File.read(master_file).chomp if File.exist?(master_file)
|
114
|
-
empty = server_info
|
129
|
+
empty = server_info !~ /:\d+/
|
115
130
|
break if empty
|
116
131
|
sleep 1
|
117
132
|
end
|
@@ -125,7 +140,7 @@ Then /^the redis master of the beetle handler should be "([^\"]*)"$/ do |redis_n
|
|
125
140
|
config.queue(:echo)
|
126
141
|
config.message(:echo)
|
127
142
|
end
|
128
|
-
|
143
|
+
assert_match /#{TestDaemons::Redis[redis_name].ip_with_port}/, client.rpc(:echo, 'nil').second
|
129
144
|
end
|
130
145
|
|
131
146
|
Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
|
@@ -157,7 +172,7 @@ end
|
|
157
172
|
|
158
173
|
Given /^an immediate master switch is initiated and responds with (\d+)$/ do |response_code|
|
159
174
|
response = TestDaemons::RedisConfigurationServer.initiate_master_switch
|
160
|
-
assert_equal response_code, response.code
|
175
|
+
assert_equal response_code, response.code, "unexpected response code #{response.code}, message: #{response.body}"
|
161
176
|
sleep 1
|
162
177
|
end
|
163
178
|
|
@@ -24,9 +24,16 @@ Daemons.run_proc("beetle_handler", :log_output => true, :dir_mode => :normal, :d
|
|
24
24
|
client = Beetle::Client.new.configure :auto_delete => true do |config|
|
25
25
|
config.queue(:echo)
|
26
26
|
config.message(:echo)
|
27
|
-
config.handler(:echo)
|
27
|
+
config.handler(:echo) do |message|
|
28
|
+
begin
|
29
|
+
client.deduplication_store.redis.server
|
30
|
+
rescue
|
31
|
+
master_file_content = File.read(Beetle.config.redis_server)
|
32
|
+
"no redis master: exception: #{$!.class}(#{$!}), master_file: '#{master_file_content}'"
|
33
|
+
end
|
34
|
+
end
|
28
35
|
end
|
29
36
|
client.listen do
|
30
|
-
puts "Started beetle handler"
|
37
|
+
puts "Started beetle handler for system: #{Beetle.config.system_name}"
|
31
38
|
end
|
32
39
|
end
|
data/features/support/env.rb
CHANGED
@@ -7,6 +7,7 @@ World do
|
|
7
7
|
end
|
8
8
|
|
9
9
|
Before do
|
10
|
+
cleanup_master_files
|
10
11
|
`ruby features/support/system_notification_logger start`
|
11
12
|
end
|
12
13
|
|
@@ -23,8 +24,6 @@ def cleanup_test_env
|
|
23
24
|
TestDaemons::RedisConfigurationServer.stop
|
24
25
|
|
25
26
|
`ruby features/support/beetle_handler stop`
|
26
|
-
redis_master_files = tmp_path + "/redis-master-*"
|
27
|
-
`rm -f #{redis_master_files}`
|
28
27
|
|
29
28
|
`ruby features/support/system_notification_logger stop`
|
30
29
|
# `rm -f #{system_notification_log_path}`
|
@@ -32,6 +31,11 @@ def cleanup_test_env
|
|
32
31
|
TestDaemons::Redis.stop_all
|
33
32
|
end
|
34
33
|
|
34
|
+
def cleanup_master_files
|
35
|
+
redis_master_files = tmp_path + "/redis-master-*"
|
36
|
+
`rm -f #{redis_master_files}`
|
37
|
+
end
|
38
|
+
|
35
39
|
def redis_master_file(client_name)
|
36
40
|
tmp_path + "/redis-master-#{client_name}"
|
37
41
|
end
|
@@ -12,10 +12,10 @@ module TestDaemons
|
|
12
12
|
@@confidence_level = 100
|
13
13
|
|
14
14
|
def self.start(redis_servers, redis_configuration_clients, confidence_level)
|
15
|
-
stop
|
16
15
|
@@redis_servers = redis_servers
|
17
16
|
@@redis_configuration_clients = redis_configuration_clients
|
18
17
|
@@confidence_level = confidence_level
|
18
|
+
stop
|
19
19
|
daemon_controller.start
|
20
20
|
end
|
21
21
|
|
@@ -27,7 +27,7 @@ module TestDaemons
|
|
27
27
|
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
28
28
|
DaemonController.new(
|
29
29
|
:identifier => "Redis configuration test server",
|
30
|
-
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
|
30
|
+
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
|
31
31
|
:ping_command => lambda{ answers_text_requests? },
|
32
32
|
:pid_file => pid_file,
|
33
33
|
:log_file => log_file,
|
@@ -86,9 +86,9 @@ module TestDaemons
|
|
86
86
|
response
|
87
87
|
end
|
88
88
|
|
89
|
-
def self.initiate_master_switch
|
89
|
+
def self.initiate_master_switch(system_name = "system")
|
90
90
|
http = Net::HTTP.new('127.0.0.1', 9650)
|
91
|
-
response = http.post
|
91
|
+
response = http.post "/initiate_master_switch?system_name=#{system_name}", ""
|
92
92
|
response
|
93
93
|
end
|
94
94
|
|
data/lib/beetle.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
$:.unshift(File.expand_path('..', __FILE__))
|
2
2
|
require 'bunny' # which bunny picks up
|
3
3
|
require 'qrack/errors' # needed by the publisher
|
4
|
-
require 'uuid4r'
|
5
4
|
require 'redis/connection/hiredis' # require *before* redis as specified in the redis-rb gem docs
|
6
5
|
require 'redis'
|
7
6
|
require 'active_support/all'
|
data/lib/beetle/configuration.rb
CHANGED
@@ -62,6 +62,11 @@ module Beetle
|
|
62
62
|
# in bunny.
|
63
63
|
attr_accessor :channel_max
|
64
64
|
|
65
|
+
# Lazy queues have the advantage of consuming a lot less memory on the broker. For backwards
|
66
|
+
# compatibility, they are disabled by default.
|
67
|
+
attr_accessor :lazy_queues_enabled
|
68
|
+
alias_method :lazy_queues_enabled?, :lazy_queues_enabled
|
69
|
+
|
65
70
|
# In contrast to RabbitMQ 2.x, RabbitMQ 3.x preserves message order when requeing a message. This can lead to
|
66
71
|
# throughput degradation (when rejected messages block the processing of other messages
|
67
72
|
# at the head of the queue) in some cases.
|
@@ -89,6 +94,11 @@ module Beetle
|
|
89
94
|
# consider this a highly experimental feature for now.
|
90
95
|
attr_accessor :publishing_timeout
|
91
96
|
|
97
|
+
# the connect/disconnect timeout in seconds for the publishing connection
|
98
|
+
# (defaults to <tt>5</tt>).
|
99
|
+
# consider this a highly experimental feature for now.
|
100
|
+
attr_accessor :publisher_connect_timeout
|
101
|
+
|
92
102
|
# Prefetch count for subscribers (defaults to 1). Setting this higher
|
93
103
|
# than 1 can potentially increase throughput, but comes at the cost of
|
94
104
|
# decreased parallelism.
|
@@ -138,7 +148,10 @@ module Beetle
|
|
138
148
|
self.dead_lettering_msg_ttl = 1000 #1 second
|
139
149
|
self.dead_lettering_read_timeout = 3 #3 seconds
|
140
150
|
|
151
|
+
self.lazy_queues_enabled = false
|
152
|
+
|
141
153
|
self.publishing_timeout = 0
|
154
|
+
self.publisher_connect_timeout = 5
|
142
155
|
self.tmpdir = "/tmp"
|
143
156
|
|
144
157
|
self.log_file = STDOUT
|
@@ -10,54 +10,64 @@ module Beetle
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def bind_dead_letter_queues!(channel, servers, target_queue, creation_keys = {})
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
channel.queue(dead_letter_queue_name, creation_keys)
|
19
|
-
|
20
|
-
logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
|
21
|
-
set_dead_letter_policies!(servers, target_queue)
|
13
|
+
if @config.dead_lettering_enabled?
|
14
|
+
dead_letter_queue_name = dead_letter_queue_name(target_queue)
|
15
|
+
logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
|
16
|
+
channel.queue(dead_letter_queue_name, creation_keys)
|
17
|
+
end
|
22
18
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
19
|
+
if @config.dead_lettering_enabled?
|
20
|
+
logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
|
21
|
+
end
|
22
|
+
set_queue_policies!(servers, target_queue)
|
23
|
+
|
24
|
+
if @config.dead_lettering_enabled?
|
25
|
+
logger.debug("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
|
26
|
+
set_queue_policies!(
|
27
|
+
servers,
|
28
|
+
dead_letter_queue_name,
|
29
|
+
:message_ttl => @config.dead_lettering_msg_ttl,
|
30
|
+
:routing_key => target_queue
|
31
|
+
)
|
32
|
+
end
|
30
33
|
end
|
31
34
|
|
32
|
-
def
|
33
|
-
servers.each { |server|
|
35
|
+
def set_queue_policies!(servers, queue_name, options={})
|
36
|
+
servers.each { |server| set_queue_policy!(server, queue_name, options) }
|
34
37
|
end
|
35
38
|
|
36
|
-
def
|
39
|
+
def set_queue_policy!(server, queue_name, options={})
|
37
40
|
raise ArgumentError.new("server missing") if server.blank?
|
38
41
|
raise ArgumentError.new("queue name missing") if queue_name.blank?
|
39
42
|
|
43
|
+
return unless @config.dead_lettering_enabled? || @config.lazy_queues_enabled?
|
44
|
+
|
40
45
|
vhost = CGI.escape(@config.vhost)
|
41
46
|
request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
|
42
47
|
request = Net::HTTP::Put.new(request_url)
|
43
48
|
|
49
|
+
# set up queue policy
|
50
|
+
definition = {}
|
51
|
+
if @config.dead_lettering_enabled?
|
52
|
+
definition["dead-letter-routing-key"] = dead_letter_routing_key(queue_name, options)
|
53
|
+
definition["dead-letter-exchange"] = ""
|
54
|
+
definition["message-ttl"] = options[:message_ttl] if options[:message_ttl]
|
55
|
+
end
|
56
|
+
|
57
|
+
definition["queue-mode"] = "lazy" if @config.lazy_queues_enabled?
|
58
|
+
|
44
59
|
request_body = {
|
45
60
|
"pattern" => "^#{queue_name}$",
|
46
61
|
"priority" => 1,
|
47
62
|
"apply-to" => "queues",
|
48
|
-
"definition" =>
|
49
|
-
"dead-letter-routing-key" => dead_letter_routing_key(queue_name, options),
|
50
|
-
"dead-letter-exchange" => ""
|
51
|
-
}
|
63
|
+
"definition" => definition,
|
52
64
|
}
|
53
65
|
|
54
|
-
request_body["definition"].merge!("message-ttl" => options[:message_ttl]) if options[:message_ttl]
|
55
|
-
|
56
66
|
response = run_rabbit_http_request(request_url, request) do |http|
|
57
67
|
http.request(request, request_body.to_json)
|
58
68
|
end
|
59
69
|
|
60
|
-
|
70
|
+
unless %w(200 201 204).include?(response.code)
|
61
71
|
log_error("Failed to create policy for queue #{queue_name}", response)
|
62
72
|
raise FailedRabbitRequest.new("Could not create policy")
|
63
73
|
end
|
@@ -142,13 +142,30 @@ module Beetle
|
|
142
142
|
@last_time_master_file_changed != File.mtime(@config.redis_server)
|
143
143
|
end
|
144
144
|
|
145
|
-
# set current redis master from server:port string contained in the redis master
|
145
|
+
# set current redis master from server:port string contained in the redis master for our system
|
146
146
|
def set_current_redis_master_from_master_file
|
147
147
|
@last_time_master_file_changed = File.mtime(@config.redis_server)
|
148
|
-
server_string = read_master_file
|
148
|
+
server_string = extract_redis_master(read_master_file)
|
149
149
|
@current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
|
150
150
|
end
|
151
151
|
|
152
|
+
# extract redis master from file content and return the server for our system
|
153
|
+
def extract_redis_master(text)
|
154
|
+
system_name = @config.system_name
|
155
|
+
redis_master = ""
|
156
|
+
text.each_line do |line|
|
157
|
+
parts = line.split('/')
|
158
|
+
case parts.size
|
159
|
+
when 1
|
160
|
+
redis_master = parts[0]
|
161
|
+
when 2
|
162
|
+
name, master = parts
|
163
|
+
redis_master = master if name == system_name
|
164
|
+
end
|
165
|
+
end
|
166
|
+
redis_master
|
167
|
+
end
|
168
|
+
|
152
169
|
# server:port string from the redis master file
|
153
170
|
def read_master_file
|
154
171
|
File.read(@config.redis_server).chomp
|
data/lib/beetle/message.rb
CHANGED
@@ -146,8 +146,16 @@ module Beetle
|
|
146
146
|
end
|
147
147
|
|
148
148
|
# generate uuid for publishing
|
149
|
-
|
150
|
-
|
149
|
+
begin
|
150
|
+
require "uuid4r"
|
151
|
+
def self.generate_uuid
|
152
|
+
UUID4R::uuid(4)
|
153
|
+
end
|
154
|
+
rescue LoadError
|
155
|
+
require "securerandom"
|
156
|
+
def self.generate_uuid
|
157
|
+
SecureRandom.uuid
|
158
|
+
end
|
151
159
|
end
|
152
160
|
|
153
161
|
# whether the publisher has tried sending this message to two servers
|
data/lib/beetle/publisher.rb
CHANGED
@@ -159,15 +159,16 @@ module Beetle
|
|
159
159
|
|
160
160
|
def new_bunny
|
161
161
|
b = Bunny.new(
|
162
|
-
:host
|
163
|
-
:port
|
164
|
-
:logging
|
165
|
-
:user
|
166
|
-
:pass
|
167
|
-
:vhost
|
168
|
-
:frame_max
|
169
|
-
:channel_max
|
170
|
-
:socket_timeout
|
162
|
+
:host => current_host,
|
163
|
+
:port => current_port,
|
164
|
+
:logging => !!@options[:logging],
|
165
|
+
:user => @client.config.user,
|
166
|
+
:pass => @client.config.password,
|
167
|
+
:vhost => @client.config.vhost,
|
168
|
+
:frame_max => @client.config.frame_max,
|
169
|
+
:channel_max => @client.config.channel_max,
|
170
|
+
:socket_timeout => @client.config.publishing_timeout,
|
171
|
+
:connect_timeout => @client.config.publisher_connect_timeout,
|
171
172
|
:spec => '09')
|
172
173
|
b.start
|
173
174
|
b
|
@@ -224,7 +225,8 @@ module Beetle
|
|
224
225
|
|
225
226
|
def stop!(exception=nil)
|
226
227
|
return unless bunny?
|
227
|
-
|
228
|
+
timeout = @client.config.publishing_timeout + @client.config.publisher_connect_timeout + 1
|
229
|
+
Beetle::Timer.timeout(timeout) do
|
228
230
|
logger.debug "Beetle: closing connection from publisher to #{server}"
|
229
231
|
if exception
|
230
232
|
bunny.__send__ :close_socket
|
data/lib/beetle/version.rb
CHANGED
data/script/console
CHANGED
@@ -10,16 +10,16 @@ OptionParser.new do |opt|
|
|
10
10
|
end
|
11
11
|
|
12
12
|
libs = " -r irb/completion"
|
13
|
-
libs << %( -r
|
13
|
+
libs << %( -r rubygems)
|
14
14
|
libs << %( -r #{File.expand_path("../../lib/beetle.rb",__FILE__)})
|
15
15
|
|
16
16
|
if options[:debugger]
|
17
17
|
begin
|
18
|
-
require '
|
19
|
-
libs << " -r
|
18
|
+
require 'byebug'
|
19
|
+
libs << " -r byebug"
|
20
20
|
puts "=> Debugger enabled"
|
21
21
|
rescue Exception
|
22
|
-
puts "You need to install
|
22
|
+
puts "You need to install byebug to run the console in debugging mode. With gems, use 'gem install byebug'"
|
23
23
|
exit
|
24
24
|
end
|
25
25
|
end
|
@@ -3,18 +3,20 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
|
3
3
|
module Beetle
|
4
4
|
class SetDeadLetteringsTest < Minitest::Test
|
5
5
|
def setup
|
6
|
-
@
|
6
|
+
@config = Configuration.new
|
7
|
+
@dead_lettering = DeadLettering.new(@config)
|
8
|
+
@config.dead_lettering_enabled = true
|
7
9
|
end
|
8
10
|
|
9
11
|
test "creates a dead letter queue for each server" do
|
10
12
|
servers = %w(a b)
|
11
13
|
|
12
|
-
@dead_lettering.expects(:
|
14
|
+
@dead_lettering.expects(:set_queue_policy!).
|
13
15
|
with("a", "QUEUE_NAME", :message_ttl => 10000)
|
14
|
-
@dead_lettering.expects(:
|
16
|
+
@dead_lettering.expects(:set_queue_policy!).
|
15
17
|
with("b", "QUEUE_NAME", :message_ttl => 10000)
|
16
18
|
|
17
|
-
@dead_lettering.
|
19
|
+
@dead_lettering.set_queue_policies!(servers, "QUEUE_NAME", :message_ttl => 10000)
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
@@ -25,21 +27,22 @@ module Beetle
|
|
25
27
|
@config = Configuration.new
|
26
28
|
@config.logger = Logger.new("/dev/null")
|
27
29
|
@dead_lettering = DeadLettering.new(@config)
|
30
|
+
@config.dead_lettering_enabled = true
|
28
31
|
end
|
29
32
|
|
30
33
|
test "raises exception when queue name wasn't specified" do
|
31
34
|
assert_raises ArgumentError do
|
32
|
-
@dead_lettering.
|
35
|
+
@dead_lettering.set_queue_policy!(@server, "")
|
33
36
|
end
|
34
37
|
end
|
35
38
|
|
36
39
|
test "raises exception when no server was specified" do
|
37
40
|
assert_raises ArgumentError do
|
38
|
-
@dead_lettering.
|
41
|
+
@dead_lettering.set_queue_policy!("", @queue_name)
|
39
42
|
end
|
40
43
|
end
|
41
44
|
|
42
|
-
test "creates a policy by posting to the rabbitmq" do
|
45
|
+
test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
|
43
46
|
stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
44
47
|
.with(basic_auth: ['guest', 'guest'])
|
45
48
|
.with(:body => {
|
@@ -52,7 +55,24 @@ module Beetle
|
|
52
55
|
}}.to_json)
|
53
56
|
.to_return(:status => 204)
|
54
57
|
|
55
|
-
@dead_lettering.
|
58
|
+
@dead_lettering.set_queue_policy!(@server, @queue_name)
|
59
|
+
end
|
60
|
+
|
61
|
+
test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
|
62
|
+
@config.lazy_queues_enabled = true
|
63
|
+
@config.dead_lettering_enabled = false
|
64
|
+
stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
65
|
+
.with(basic_auth: ['guest', 'guest'])
|
66
|
+
.with(:body => {
|
67
|
+
"pattern" => "^QUEUE_NAME$",
|
68
|
+
"priority" => 1,
|
69
|
+
"apply-to" => "queues",
|
70
|
+
"definition" => {
|
71
|
+
"queue-mode" => "lazy"
|
72
|
+
}}.to_json)
|
73
|
+
.to_return(:status => 204)
|
74
|
+
|
75
|
+
@dead_lettering.set_queue_policy!(@server, @queue_name)
|
56
76
|
end
|
57
77
|
|
58
78
|
test "raises exception when policy couldn't successfully be created" do
|
@@ -61,7 +81,7 @@ module Beetle
|
|
61
81
|
.to_return(:status => [405])
|
62
82
|
|
63
83
|
assert_raises DeadLettering::FailedRabbitRequest do
|
64
|
-
@dead_lettering.
|
84
|
+
@dead_lettering.set_queue_policy!(@server, @queue_name)
|
65
85
|
end
|
66
86
|
end
|
67
87
|
|
@@ -79,7 +99,7 @@ module Beetle
|
|
79
99
|
}}.to_json)
|
80
100
|
.to_return(:status => 204)
|
81
101
|
|
82
|
-
@dead_lettering.
|
102
|
+
@dead_lettering.set_queue_policy!(@server, @queue_name, :message_ttl => 10000)
|
83
103
|
end
|
84
104
|
|
85
105
|
test "properly encodes the vhost from the configuration" do
|
@@ -97,7 +117,7 @@ module Beetle
|
|
97
117
|
|
98
118
|
@config.vhost = "foo/"
|
99
119
|
|
100
|
-
@dead_lettering.
|
120
|
+
@dead_lettering.set_queue_policy!(@server, @queue_name)
|
101
121
|
end
|
102
122
|
end
|
103
123
|
|
@@ -110,9 +130,10 @@ module Beetle
|
|
110
130
|
@servers = ["localhost:55672"]
|
111
131
|
end
|
112
132
|
|
113
|
-
test "is
|
133
|
+
test "is does not call out to rabbit if neither dead lettering nor lazy queues are enabled" do
|
134
|
+
@config.dead_lettering_enabled = false
|
114
135
|
channel = stub('channel')
|
115
|
-
@dead_lettering.expects(:
|
136
|
+
@dead_lettering.expects(:run_rabbit_http_request).never
|
116
137
|
@dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)
|
117
138
|
end
|
118
139
|
|
@@ -122,8 +143,8 @@ module Beetle
|
|
122
143
|
channel = stub('channel')
|
123
144
|
|
124
145
|
channel.expects(:queue).with("#{@queue_name}_dead_letter", {})
|
125
|
-
@dead_lettering.expects(:
|
126
|
-
@dead_lettering.expects(:
|
146
|
+
@dead_lettering.expects(:set_queue_policies!).with(@servers, @queue_name)
|
147
|
+
@dead_lettering.expects(:set_queue_policies!).with(@servers, "#{@queue_name}_dead_letter",
|
127
148
|
:routing_key => @queue_name,
|
128
149
|
:message_ttl => 1000
|
129
150
|
)
|
@@ -43,6 +43,7 @@ module Beetle
|
|
43
43
|
class RedisServerFileTest < Minitest::Test
|
44
44
|
def setup
|
45
45
|
@original_redis_server = Beetle.config.redis_server
|
46
|
+
@original_system_name = Beetle.config.system_name
|
46
47
|
@store = DeduplicationStore.new
|
47
48
|
@server_string = "my_test_host_from_file:6379"
|
48
49
|
Beetle.config.redis_server = redis_test_master_file(@server_string)
|
@@ -50,6 +51,7 @@ module Beetle
|
|
50
51
|
|
51
52
|
def teardown
|
52
53
|
Beetle.config.redis_server = @original_redis_server
|
54
|
+
Beetle.config.system_name = @original_system_name
|
53
55
|
end
|
54
56
|
|
55
57
|
test "redis should match the redis master file" do
|
@@ -71,6 +73,17 @@ module Beetle
|
|
71
73
|
assert_raises(Errno::ENOENT) { @store.redis_master_from_master_file }
|
72
74
|
end
|
73
75
|
|
76
|
+
test "should retrieve the redis master for the configured system name if the master file contains a mapping for it" do
|
77
|
+
redis_test_master_file("blabber/localhost:2\nblubber/localhost:1")
|
78
|
+
Beetle.config.system_name = "blubber"
|
79
|
+
assert_equal "localhost:1", @store.redis.server
|
80
|
+
end
|
81
|
+
|
82
|
+
test "should retrieve the redis master for the default system name if the master file contains a simple host:port entry" do
|
83
|
+
redis_test_master_file("localhost:2\nblubber/localhost:1")
|
84
|
+
assert_equal "localhost:2", @store.redis.server
|
85
|
+
end
|
86
|
+
|
74
87
|
private
|
75
88
|
def redis_test_master_file(server_string)
|
76
89
|
tmp_dir = File.expand_path("../../../tmp", __FILE__)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stefan Kaes
|
@@ -12,36 +12,22 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2019-02-
|
15
|
+
date: 2019-02-28 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
|
-
- !ruby/object:Gem::Dependency
|
18
|
-
name: uuid4r
|
19
|
-
requirement: !ruby/object:Gem::Requirement
|
20
|
-
requirements:
|
21
|
-
- - ">="
|
22
|
-
- !ruby/object:Gem::Version
|
23
|
-
version: 0.1.2
|
24
|
-
type: :runtime
|
25
|
-
prerelease: false
|
26
|
-
version_requirements: !ruby/object:Gem::Requirement
|
27
|
-
requirements:
|
28
|
-
- - ">="
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
version: 0.1.2
|
31
17
|
- !ruby/object:Gem::Dependency
|
32
18
|
name: bunny
|
33
19
|
requirement: !ruby/object:Gem::Requirement
|
34
20
|
requirements:
|
35
21
|
- - "~>"
|
36
22
|
- !ruby/object:Gem::Version
|
37
|
-
version: 0.7.
|
23
|
+
version: 0.7.12
|
38
24
|
type: :runtime
|
39
25
|
prerelease: false
|
40
26
|
version_requirements: !ruby/object:Gem::Requirement
|
41
27
|
requirements:
|
42
28
|
- - "~>"
|
43
29
|
- !ruby/object:Gem::Version
|
44
|
-
version: 0.7.
|
30
|
+
version: 0.7.12
|
45
31
|
- !ruby/object:Gem::Dependency
|
46
32
|
name: redis
|
47
33
|
requirement: !ruby/object:Gem::Requirement
|
@@ -112,6 +98,20 @@ dependencies:
|
|
112
98
|
- - ">="
|
113
99
|
- !ruby/object:Gem::Version
|
114
100
|
version: 2.3.4
|
101
|
+
- !ruby/object:Gem::Dependency
|
102
|
+
name: uuid4r
|
103
|
+
requirement: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: 0.1.2
|
108
|
+
type: :development
|
109
|
+
prerelease: false
|
110
|
+
version_requirements: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: 0.1.2
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: activerecord
|
117
117
|
requirement: !ruby/object:Gem::Requirement
|
@@ -328,7 +328,6 @@ files:
|
|
328
328
|
- examples/redundant.rb
|
329
329
|
- examples/rpc.rb
|
330
330
|
- examples/simple.rb
|
331
|
-
- features/#redis_auto_failover.feature#
|
332
331
|
- features/README.rdoc
|
333
332
|
- features/redis_auto_failover.feature
|
334
333
|
- features/step_definitions/redis_auto_failover_steps.rb
|
@@ -351,7 +350,6 @@ files:
|
|
351
350
|
- lib/beetle/publisher.rb
|
352
351
|
- lib/beetle/r_c.rb
|
353
352
|
- lib/beetle/redis_ext.rb
|
354
|
-
- lib/beetle/redis_master_file.rb
|
355
353
|
- lib/beetle/subscriber.rb
|
356
354
|
- lib/beetle/version.rb
|
357
355
|
- script/console
|
@@ -369,13 +367,13 @@ files:
|
|
369
367
|
- test/beetle/publisher_test.rb
|
370
368
|
- test/beetle/r_c_test.rb
|
371
369
|
- test/beetle/redis_ext_test.rb
|
372
|
-
- test/beetle/redis_master_file_test.rb
|
373
370
|
- test/beetle/subscriber_test.rb
|
374
371
|
- test/beetle_test.rb
|
375
372
|
- test/test_helper.rb
|
376
373
|
homepage: https://xing.github.com/beetle/
|
377
374
|
licenses: []
|
378
|
-
metadata:
|
375
|
+
metadata:
|
376
|
+
changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
|
379
377
|
post_install_message:
|
380
378
|
rdoc_options:
|
381
379
|
- "--charset=UTF-8"
|
@@ -403,7 +401,6 @@ test_files:
|
|
403
401
|
- test/beetle/deduplication_store_test.rb
|
404
402
|
- test/beetle/handler_test.rb
|
405
403
|
- test/beetle/beetle_test.rb
|
406
|
-
- test/beetle/redis_master_file_test.rb
|
407
404
|
- test/beetle/configuration_test.rb
|
408
405
|
- test/beetle/subscriber_test.rb
|
409
406
|
- test/beetle/message/settings_test.rb
|
@@ -1,165 +0,0 @@
|
|
1
|
-
Feature: Redis auto failover
|
2
|
-
In order to eliminate a single point of failure
|
3
|
-
Beetle handlers should automatically switch to a new redis master in case of a redis master failure
|
4
|
-
|
5
|
-
Background:
|
6
|
-
Given a redis server "redis-1" exists as master
|
7
|
-
And a redis server "redis-2" exists as slave of "redis-1"
|
8
|
-
|
9
|
-
Scenario: Successful redis master switch
|
10
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
11
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
12
|
-
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
13
|
-
And a beetle handler using the redis-master file from "rc-client-1" exists
|
14
|
-
And redis server "redis-1" is down
|
15
|
-
And the retry timeout for the redis master check is reached
|
16
|
-
Then a system notification for "redis-1" not being available should be sent
|
17
|
-
And the role of redis server "redis-2" should be "master"
|
18
|
-
And the redis master file of the redis configuration server should contain "redis-2"
|
19
|
-
And the redis master of "rc-client-1" should be "redis-2"
|
20
|
-
And the redis master of "rc-client-2" should be "redis-2"
|
21
|
-
And the redis master of the beetle handler should be "redis-2"
|
22
|
-
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
23
|
-
Given a redis server "redis-1" exists as master
|
24
|
-
Then the role of redis server "redis-1" should be "slave"
|
25
|
-
|
26
|
-
Scenario: Not all redis configuration clients available (switch possible using confidence level 50)
|
27
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" and confidence level "50" exists
|
28
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
29
|
-
And a beetle handler using the redis-master file from "rc-client-1" exists
|
30
|
-
And redis configuration client "rc-client-2" is down
|
31
|
-
And redis server "redis-1" is down
|
32
|
-
And the retry timeout for the redis master check is reached
|
33
|
-
Then a system notification for "redis-1" not being available should be sent
|
34
|
-
And the role of redis server "redis-2" should be "master"
|
35
|
-
And the redis master file of the redis configuration server should contain "redis-2"
|
36
|
-
And the redis master of "rc-client-1" should be "redis-2"
|
37
|
-
And the redis master of the beetle handler should be "redis-2"
|
38
|
-
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
39
|
-
Given a redis server "redis-1" exists as master
|
40
|
-
Then the role of redis server "redis-1" should be "slave"
|
41
|
-
|
42
|
-
Scenario: Redis master only temporarily down (no switch necessary)
|
43
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
44
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
45
|
-
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
46
|
-
And a beetle handler using the redis-master file from "rc-client-1" exists
|
47
|
-
And redis server "redis-1" is down for less seconds than the retry timeout for the redis master check
|
48
|
-
And the retry timeout for the redis master check is reached
|
49
|
-
Then the role of redis server "redis-1" should be "master"
|
50
|
-
Then the role of redis server "redis-2" should be "slave"
|
51
|
-
And the redis master of "rc-client-1" should be "redis-1"
|
52
|
-
And the redis master of "rc-client-2" should be "redis-1"
|
53
|
-
And the redis master of the beetle handler should be "redis-1"
|
54
|
-
|
55
|
-
Scenario: Not all redis configuration clients available (no switch possible)
|
56
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
57
|
-
And redis server "redis-1" is down
|
58
|
-
And the retry timeout for the redis master check is reached
|
59
|
-
Then the role of redis server "redis-2" should be "slave"
|
60
|
-
|
61
|
-
Scenario: No redis slave available to become new master (no switch possible)
|
62
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
63
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
64
|
-
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
65
|
-
And redis server "redis-1" is down
|
66
|
-
And redis server "redis-2" is down
|
67
|
-
And the retry timeout for the redis master check is reached
|
68
|
-
Then the redis master of "rc-client-1" should be "redis-1"
|
69
|
-
And the redis master of "rc-client-2" should be "redis-1"
|
70
|
-
And a system notification for no slave available to become new master should be sent
|
71
|
-
|
72
|
-
Scenario: Redis configuration client starts while no redis master available
|
73
|
-
Given redis server "redis-1" is down
|
74
|
-
And redis server "redis-2" is down
|
75
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
76
|
-
And the retry timeout for the redis master determination is reached
|
77
|
-
Then the redis master of "rc-client-1" should be undefined
|
78
|
-
|
79
|
-
Scenario: Redis configuration client starts while no redis master available but master file exists
|
80
|
-
Given redis server "redis-1" is down
|
81
|
-
And redis server "redis-2" is down
|
82
|
-
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
83
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
84
|
-
And the retry timeout for the redis master determination is reached
|
85
|
-
Then the redis master of "rc-client-1" should be undefined
|
86
|
-
|
87
|
-
Scenario: Redis configuration client starts while both redis servers are master
|
88
|
-
Given redis server "redis-2" is master
|
89
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
90
|
-
Then the redis master of "rc-client-1" should be undefined
|
91
|
-
|
92
|
-
Scenario: Redis configuration client starts while both redis servers are master but master file exists
|
93
|
-
Given redis server "redis-2" is master
|
94
|
-
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
95
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
96
|
-
Then the redis master of "rc-client-1" should be "redis-1"
|
97
|
-
|
98
|
-
Scenario: Redis configuration client starts while both redis servers are slave
|
99
|
-
Given a redis server "redis-3" exists as master
|
100
|
-
And redis server "redis-1" is slave of "redis-3"
|
101
|
-
And redis server "redis-2" is slave of "redis-3"
|
102
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
103
|
-
Then the redis master of "rc-client-1" should be undefined
|
104
|
-
|
105
|
-
Scenario: Redis configuration client starts while both redis servers are slave but master file exists
|
106
|
-
Given a redis server "redis-3" exists as master
|
107
|
-
And redis server "redis-1" is slave of "redis-3"
|
108
|
-
And redis server "redis-2" is slave of "redis-3"
|
109
|
-
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
110
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
111
|
-
Then the redis master of "rc-client-1" should be undefined
|
112
|
-
|
113
|
-
Scenario: Redis configuration client starts while there is a redis master but no slave
|
114
|
-
Given redis server "redis-2" is down
|
115
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
116
|
-
Then the redis master of "rc-client-1" should be undefined
|
117
|
-
|
118
|
-
Scenario: Redis configuration client starts while there is a redis master but no slave but master file exists
|
119
|
-
Given redis server "redis-2" is down
|
120
|
-
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
121
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
122
|
-
Then the redis master of "rc-client-1" should be "redis-1"
|
123
|
-
|
124
|
-
Scenario: Redis configuation server should embed a http server
|
125
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
126
|
-
Then the redis configuration server should answer http requests
|
127
|
-
|
128
|
-
Scenario: Accelerated redis master switch when master is down
|
129
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
130
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
131
|
-
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
132
|
-
And a beetle handler using the redis-master file from "rc-client-1" exists
|
133
|
-
And redis server "redis-1" is down
|
134
|
-
And an immediate master switch is initiated and responds with 201
|
135
|
-
Then a system notification for "redis-1" not being available should be sent
|
136
|
-
And the role of redis server "redis-2" should be "master"
|
137
|
-
And the redis master of "rc-client-1" should be "redis-2"
|
138
|
-
And the redis master of "rc-client-2" should be "redis-2"
|
139
|
-
And the redis master of the beetle handler should be "redis-2"
|
140
|
-
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
141
|
-
Given a redis server "redis-1" exists as master
|
142
|
-
Then the role of redis server "redis-1" should be "slave"
|
143
|
-
|
144
|
-
Scenario: Accelerated redis master switch when master is up
|
145
|
-
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
146
|
-
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
147
|
-
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
148
|
-
And a beetle handler using the redis-master file from "rc-client-1" exists
|
149
|
-
And an immediate master switch is initiated and responds with 200
|
150
|
-
Then the role of redis server "redis-1" should be "master"
|
151
|
-
And the redis master of "rc-client-1" should be "redis-1"
|
152
|
-
And the redis master of "rc-client-2" should be "redis-1"
|
153
|
-
And the redis master of the beetle handler should be "redis-1"
|
154
|
-
And the role of redis server "redis-2" should be "slave"
|
155
|
-
|
156
|
-
# Scenario: Running the system for a few seconds to perform manual testing
|
157
|
-
# Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
158
|
-
# And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
159
|
-
# And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
160
|
-
# And a beetle handler using the redis-master file from "rc-client-1" exists
|
161
|
-
# And the redis master of "rc-client-1" should be "redis-1"
|
162
|
-
# And the redis master of "rc-client-2" should be "redis-1"
|
163
|
-
# And the redis master of the beetle handler should be "redis-1"
|
164
|
-
# And the role of redis server "redis-2" should be "slave"
|
165
|
-
# Then the system can run for a while without dying
|
@@ -1,30 +0,0 @@
|
|
1
|
-
module Beetle
|
2
|
-
module RedisMasterFile #:nodoc:
|
3
|
-
private
|
4
|
-
def master_file_exists?
|
5
|
-
File.exist?(master_file)
|
6
|
-
end
|
7
|
-
|
8
|
-
def redis_master_from_master_file
|
9
|
-
redis.find(read_redis_master_file)
|
10
|
-
end
|
11
|
-
|
12
|
-
def clear_redis_master_file
|
13
|
-
write_redis_master_file("")
|
14
|
-
end
|
15
|
-
|
16
|
-
def read_redis_master_file
|
17
|
-
File.read(master_file).chomp
|
18
|
-
end
|
19
|
-
|
20
|
-
def write_redis_master_file(redis_server_string)
|
21
|
-
logger.warn "Writing '#{redis_server_string}' to redis master file '#{master_file}'"
|
22
|
-
File.open(master_file, "w"){|f| f.puts redis_server_string }
|
23
|
-
end
|
24
|
-
|
25
|
-
def master_file
|
26
|
-
config.redis_server
|
27
|
-
end
|
28
|
-
|
29
|
-
end
|
30
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
2
|
-
|
3
|
-
module Beetle
|
4
|
-
|
5
|
-
class RedisMasterFileTest < Minitest::Test
|
6
|
-
include Logging
|
7
|
-
include RedisMasterFile
|
8
|
-
|
9
|
-
def setup
|
10
|
-
File.open(master_file, "w"){|f| f.puts "localhost:6379"}
|
11
|
-
end
|
12
|
-
|
13
|
-
def teardown
|
14
|
-
File.unlink(master_file) if File.exist?(master_file)
|
15
|
-
end
|
16
|
-
|
17
|
-
test "should be able to check existence" do
|
18
|
-
assert master_file_exists?
|
19
|
-
File.unlink(master_file)
|
20
|
-
assert !master_file_exists?
|
21
|
-
end
|
22
|
-
|
23
|
-
test "should be able to read and write the master file"do
|
24
|
-
write_redis_master_file("localhost:6380")
|
25
|
-
assert_equal "localhost:6380", read_redis_master_file
|
26
|
-
end
|
27
|
-
|
28
|
-
test "should be able to clear the master file" do
|
29
|
-
logger.expects(:warn)
|
30
|
-
clear_redis_master_file
|
31
|
-
assert_equal "", read_redis_master_file
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
def master_file
|
36
|
-
"/tmp/mumu.txt"
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|