beetle 2.1.0 → 2.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/RELEASE_NOTES.rdoc +4 -0
- data/features/#redis_auto_failover.feature# +165 -0
- data/features/redis_auto_failover.feature +16 -0
- data/features/step_definitions/redis_auto_failover_steps.rb +6 -2
- data/features/support/test_daemons/redis_configuration_server.rb +4 -2
- data/lib/beetle/version.rb +1 -1
- metadata +16 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 131e66f06e511b18d6538b46f614d02a34408b113229f382c9f42e64e896b8c2
|
|
4
|
+
data.tar.gz: f2f8229336ee6c0f102145a45d58defb15edb8b34fa89728dc5c2a1083625a0e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7fcb6d6966b8de7f7ef74085815b4bf2a5d6e8970a87c5a16af556660fe7c19cec8adb65234bcc027a8352796c5e2ed42d9bc8547059883947c9ec57bbf3cea4
|
|
7
|
+
data.tar.gz: 85e8f2289c30aca215bf0c9cf7662ca53864f902f5493b19fc090645f038b69021f1a71e684d62f58324c2b6cbd1184e3c4c51465462c5aa32f28b262129c62d
|
data/RELEASE_NOTES.rdoc
CHANGED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
Feature: Redis auto failover
|
|
2
|
+
In order to eliminate a single point of failure
|
|
3
|
+
Beetle handlers should automatically switch to a new redis master in case of a redis master failure
|
|
4
|
+
|
|
5
|
+
Background:
|
|
6
|
+
Given a redis server "redis-1" exists as master
|
|
7
|
+
And a redis server "redis-2" exists as slave of "redis-1"
|
|
8
|
+
|
|
9
|
+
Scenario: Successful redis master switch
|
|
10
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
11
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
12
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
13
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
14
|
+
And redis server "redis-1" is down
|
|
15
|
+
And the retry timeout for the redis master check is reached
|
|
16
|
+
Then a system notification for "redis-1" not being available should be sent
|
|
17
|
+
And the role of redis server "redis-2" should be "master"
|
|
18
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
|
19
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
|
20
|
+
And the redis master of "rc-client-2" should be "redis-2"
|
|
21
|
+
And the redis master of the beetle handler should be "redis-2"
|
|
22
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
|
23
|
+
Given a redis server "redis-1" exists as master
|
|
24
|
+
Then the role of redis server "redis-1" should be "slave"
|
|
25
|
+
|
|
26
|
+
Scenario: Not all redis configuration clients available (switch possible using confidence level 50)
|
|
27
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" and confidence level "50" exists
|
|
28
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
29
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
30
|
+
And redis configuration client "rc-client-2" is down
|
|
31
|
+
And redis server "redis-1" is down
|
|
32
|
+
And the retry timeout for the redis master check is reached
|
|
33
|
+
Then a system notification for "redis-1" not being available should be sent
|
|
34
|
+
And the role of redis server "redis-2" should be "master"
|
|
35
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
|
36
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
|
37
|
+
And the redis master of the beetle handler should be "redis-2"
|
|
38
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
|
39
|
+
Given a redis server "redis-1" exists as master
|
|
40
|
+
Then the role of redis server "redis-1" should be "slave"
|
|
41
|
+
|
|
42
|
+
Scenario: Redis master only temporarily down (no switch necessary)
|
|
43
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
44
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
45
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
46
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
47
|
+
And redis server "redis-1" is down for less seconds than the retry timeout for the redis master check
|
|
48
|
+
And the retry timeout for the redis master check is reached
|
|
49
|
+
Then the role of redis server "redis-1" should be "master"
|
|
50
|
+
Then the role of redis server "redis-2" should be "slave"
|
|
51
|
+
And the redis master of "rc-client-1" should be "redis-1"
|
|
52
|
+
And the redis master of "rc-client-2" should be "redis-1"
|
|
53
|
+
And the redis master of the beetle handler should be "redis-1"
|
|
54
|
+
|
|
55
|
+
Scenario: Not all redis configuration clients available (no switch possible)
|
|
56
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
57
|
+
And redis server "redis-1" is down
|
|
58
|
+
And the retry timeout for the redis master check is reached
|
|
59
|
+
Then the role of redis server "redis-2" should be "slave"
|
|
60
|
+
|
|
61
|
+
Scenario: No redis slave available to become new master (no switch possible)
|
|
62
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
63
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
64
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
65
|
+
And redis server "redis-1" is down
|
|
66
|
+
And redis server "redis-2" is down
|
|
67
|
+
And the retry timeout for the redis master check is reached
|
|
68
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
|
69
|
+
And the redis master of "rc-client-2" should be "redis-1"
|
|
70
|
+
And a system notification for no slave available to become new master should be sent
|
|
71
|
+
|
|
72
|
+
Scenario: Redis configuration client starts while no redis master available
|
|
73
|
+
Given redis server "redis-1" is down
|
|
74
|
+
And redis server "redis-2" is down
|
|
75
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
76
|
+
And the retry timeout for the redis master determination is reached
|
|
77
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
78
|
+
|
|
79
|
+
Scenario: Redis configuration client starts while no redis master available but master file exists
|
|
80
|
+
Given redis server "redis-1" is down
|
|
81
|
+
And redis server "redis-2" is down
|
|
82
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
|
83
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
84
|
+
And the retry timeout for the redis master determination is reached
|
|
85
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
86
|
+
|
|
87
|
+
Scenario: Redis configuration client starts while both redis servers are master
|
|
88
|
+
Given redis server "redis-2" is master
|
|
89
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
90
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
91
|
+
|
|
92
|
+
Scenario: Redis configuration client starts while both redis servers are master but master file exists
|
|
93
|
+
Given redis server "redis-2" is master
|
|
94
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
|
95
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
96
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
|
97
|
+
|
|
98
|
+
Scenario: Redis configuration client starts while both redis servers are slave
|
|
99
|
+
Given a redis server "redis-3" exists as master
|
|
100
|
+
And redis server "redis-1" is slave of "redis-3"
|
|
101
|
+
And redis server "redis-2" is slave of "redis-3"
|
|
102
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
103
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
104
|
+
|
|
105
|
+
Scenario: Redis configuration client starts while both redis servers are slave but master file exists
|
|
106
|
+
Given a redis server "redis-3" exists as master
|
|
107
|
+
And redis server "redis-1" is slave of "redis-3"
|
|
108
|
+
And redis server "redis-2" is slave of "redis-3"
|
|
109
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
|
110
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
111
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
112
|
+
|
|
113
|
+
Scenario: Redis configuration client starts while there is a redis master but no slave
|
|
114
|
+
Given redis server "redis-2" is down
|
|
115
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
116
|
+
Then the redis master of "rc-client-1" should be undefined
|
|
117
|
+
|
|
118
|
+
Scenario: Redis configuration client starts while there is a redis master but no slave but master file exists
|
|
119
|
+
Given redis server "redis-2" is down
|
|
120
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
|
121
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
122
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
|
123
|
+
|
|
124
|
+
Scenario: Redis configuation server should embed a http server
|
|
125
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
126
|
+
Then the redis configuration server should answer http requests
|
|
127
|
+
|
|
128
|
+
Scenario: Accelerated redis master switch when master is down
|
|
129
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
130
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
131
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
132
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
133
|
+
And redis server "redis-1" is down
|
|
134
|
+
And an immediate master switch is initiated and responds with 201
|
|
135
|
+
Then a system notification for "redis-1" not being available should be sent
|
|
136
|
+
And the role of redis server "redis-2" should be "master"
|
|
137
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
|
138
|
+
And the redis master of "rc-client-2" should be "redis-2"
|
|
139
|
+
And the redis master of the beetle handler should be "redis-2"
|
|
140
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
|
141
|
+
Given a redis server "redis-1" exists as master
|
|
142
|
+
Then the role of redis server "redis-1" should be "slave"
|
|
143
|
+
|
|
144
|
+
Scenario: Accelerated redis master switch when master is up
|
|
145
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
146
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
147
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
148
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
149
|
+
And an immediate master switch is initiated and responds with 200
|
|
150
|
+
Then the role of redis server "redis-1" should be "master"
|
|
151
|
+
And the redis master of "rc-client-1" should be "redis-1"
|
|
152
|
+
And the redis master of "rc-client-2" should be "redis-1"
|
|
153
|
+
And the redis master of the beetle handler should be "redis-1"
|
|
154
|
+
And the role of redis server "redis-2" should be "slave"
|
|
155
|
+
|
|
156
|
+
# Scenario: Running the system for a few seconds to perform manual testing
|
|
157
|
+
# Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
158
|
+
# And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
159
|
+
# And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
|
160
|
+
# And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
161
|
+
# And the redis master of "rc-client-1" should be "redis-1"
|
|
162
|
+
# And the redis master of "rc-client-2" should be "redis-1"
|
|
163
|
+
# And the redis master of the beetle handler should be "redis-1"
|
|
164
|
+
# And the role of redis server "redis-2" should be "slave"
|
|
165
|
+
# Then the system can run for a while without dying
|
|
@@ -42,6 +42,22 @@ Feature: Redis auto failover
|
|
|
42
42
|
And the retry timeout for the redis master check is reached
|
|
43
43
|
Then the role of redis server "redis-2" should be "slave"
|
|
44
44
|
|
|
45
|
+
Scenario: Not all redis configuration clients available (successful switch using confidence level 50)
|
|
46
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" and confidence level "50" exists
|
|
47
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
48
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
|
49
|
+
And redis configuration client "rc-client-2" is down
|
|
50
|
+
And redis server "redis-1" is down
|
|
51
|
+
And the retry timeout for the redis master check is reached
|
|
52
|
+
Then a system notification for "redis-1" not being available should be sent
|
|
53
|
+
And the role of redis server "redis-2" should be "master"
|
|
54
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
|
55
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
|
56
|
+
And the redis master of the beetle handler should be "redis-2"
|
|
57
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
|
58
|
+
Given a redis server "redis-1" exists as master
|
|
59
|
+
Then the role of redis server "redis-1" should be "slave"
|
|
60
|
+
|
|
45
61
|
Scenario: No redis slave available to become new master (no switch possible)
|
|
46
62
|
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
|
47
63
|
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
|
@@ -21,9 +21,9 @@ Given /^redis server "([^\"]*)" is slave of "([^\"]*)"$/ do |redis_name, redis_m
|
|
|
21
21
|
end while !slave.slave_of?(master.host, master.port)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" exists$/ do |redis_names, redis_configuration_client_names|
|
|
24
|
+
Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" (?:and confidence level "([^\"]*)" )?exists$/ do |redis_names, redis_configuration_client_names, confidence_level|
|
|
25
25
|
redis_servers = redis_names.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
|
|
26
|
-
TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names)
|
|
26
|
+
TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names, (confidence_level || 100).to_i)
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
Given /^a redis configuration client "([^\"]*)" using redis servers "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_names|
|
|
@@ -37,6 +37,10 @@ Given /^redis server "([^\"]*)" is down$/ do |redis_name|
|
|
|
37
37
|
TestDaemons::Redis[redis_name].stop
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
+
Given /^redis configuration client "([^\"]*)" is down$/ do |redis_configuration_client_name|
|
|
41
|
+
TestDaemons::RedisConfigurationClient[redis_configuration_client_name].stop
|
|
42
|
+
end
|
|
43
|
+
|
|
40
44
|
Given /^the retry timeout for the redis master check is reached$/ do
|
|
41
45
|
basetime = Time.now
|
|
42
46
|
i = 0
|
|
@@ -9,11 +9,13 @@ module TestDaemons
|
|
|
9
9
|
|
|
10
10
|
@@redis_servers = ""
|
|
11
11
|
@@redis_configuration_clients = ""
|
|
12
|
+
@@confidence_level = 100
|
|
12
13
|
|
|
13
|
-
def self.start(redis_servers, redis_configuration_clients)
|
|
14
|
+
def self.start(redis_servers, redis_configuration_clients, confidence_level)
|
|
14
15
|
stop
|
|
15
16
|
@@redis_servers = redis_servers
|
|
16
17
|
@@redis_configuration_clients = redis_configuration_clients
|
|
18
|
+
@@confidence_level = confidence_level
|
|
17
19
|
daemon_controller.start
|
|
18
20
|
end
|
|
19
21
|
|
|
@@ -25,7 +27,7 @@ module TestDaemons
|
|
|
25
27
|
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
|
26
28
|
DaemonController.new(
|
|
27
29
|
:identifier => "Redis configuration test server",
|
|
28
|
-
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file}",
|
|
30
|
+
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
|
|
29
31
|
:ping_command => lambda{ answers_text_requests? },
|
|
30
32
|
:pid_file => pid_file,
|
|
31
33
|
:log_file => log_file,
|
data/lib/beetle/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: beetle
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.1.
|
|
4
|
+
version: 2.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Stefan Kaes
|
|
@@ -12,7 +12,7 @@ authors:
|
|
|
12
12
|
autorequire:
|
|
13
13
|
bindir: bin
|
|
14
14
|
cert_chain: []
|
|
15
|
-
date:
|
|
15
|
+
date: 2018-02-17 00:00:00.000000000 Z
|
|
16
16
|
dependencies:
|
|
17
17
|
- !ruby/object:Gem::Dependency
|
|
18
18
|
name: uuid4r
|
|
@@ -299,11 +299,11 @@ email: opensource@xing.com
|
|
|
299
299
|
executables: []
|
|
300
300
|
extensions: []
|
|
301
301
|
extra_rdoc_files:
|
|
302
|
-
- examples/README.rdoc
|
|
303
302
|
- features/README.rdoc
|
|
304
|
-
- README.rdoc
|
|
305
|
-
- REDIS_AUTO_FAILOVER.rdoc
|
|
306
303
|
- RELEASE_NOTES.rdoc
|
|
304
|
+
- examples/README.rdoc
|
|
305
|
+
- REDIS_AUTO_FAILOVER.rdoc
|
|
306
|
+
- README.rdoc
|
|
307
307
|
- MIT-LICENSE
|
|
308
308
|
files:
|
|
309
309
|
- MIT-LICENSE
|
|
@@ -328,6 +328,7 @@ files:
|
|
|
328
328
|
- examples/redundant.rb
|
|
329
329
|
- examples/rpc.rb
|
|
330
330
|
- examples/simple.rb
|
|
331
|
+
- features/#redis_auto_failover.feature#
|
|
331
332
|
- features/README.rdoc
|
|
332
333
|
- features/redis_auto_failover.feature
|
|
333
334
|
- features/step_definitions/redis_auto_failover_steps.rb
|
|
@@ -392,25 +393,25 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
392
393
|
version: 1.3.7
|
|
393
394
|
requirements: []
|
|
394
395
|
rubyforge_project:
|
|
395
|
-
rubygems_version: 2.
|
|
396
|
+
rubygems_version: 2.7.3
|
|
396
397
|
signing_key:
|
|
397
398
|
specification_version: 3
|
|
398
399
|
summary: High Availability AMQP Messaging with Redundant Queues
|
|
399
400
|
test_files:
|
|
400
|
-
- test/
|
|
401
|
-
- test/beetle/base_test.rb
|
|
402
|
-
- test/beetle/beetle_test.rb
|
|
401
|
+
- test/beetle_test.rb
|
|
403
402
|
- test/beetle/client_test.rb
|
|
404
|
-
- test/beetle/
|
|
405
|
-
- test/beetle/dead_lettering_test.rb
|
|
403
|
+
- test/beetle/amqp_gem_behavior_test.rb
|
|
406
404
|
- test/beetle/deduplication_store_test.rb
|
|
407
405
|
- test/beetle/handler_test.rb
|
|
406
|
+
- test/beetle/beetle_test.rb
|
|
407
|
+
- test/beetle/redis_master_file_test.rb
|
|
408
|
+
- test/beetle/configuration_test.rb
|
|
409
|
+
- test/beetle/subscriber_test.rb
|
|
408
410
|
- test/beetle/message/settings_test.rb
|
|
411
|
+
- test/beetle/redis_ext_test.rb
|
|
412
|
+
- test/beetle/dead_lettering_test.rb
|
|
409
413
|
- test/beetle/message_test.rb
|
|
410
414
|
- test/beetle/publisher_test.rb
|
|
411
415
|
- test/beetle/r_c_test.rb
|
|
412
|
-
- test/beetle/
|
|
413
|
-
- test/beetle/redis_master_file_test.rb
|
|
414
|
-
- test/beetle/subscriber_test.rb
|
|
415
|
-
- test/beetle_test.rb
|
|
416
|
+
- test/beetle/base_test.rb
|
|
416
417
|
- test/test_helper.rb
|