beetle 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +20 -0
- data/REDIS_AUTO_FAILOVER.rdoc +124 -0
- data/RELEASE_NOTES.rdoc +50 -0
- data/Rakefile +113 -0
- data/beetle.gemspec +3 -3
- data/features/README.rdoc +23 -0
- data/features/redis_auto_failover.feature +105 -0
- data/features/step_definitions/redis_auto_failover_steps.rb +133 -0
- data/features/support/beetle_handler +32 -0
- data/features/support/env.rb +48 -0
- data/features/support/system_notification_logger +31 -0
- data/features/support/test_daemons/redis.conf.erb +189 -0
- data/features/support/test_daemons/redis.rb +186 -0
- data/features/support/test_daemons/redis_configuration_client.rb +64 -0
- data/features/support/test_daemons/redis_configuration_server.rb +52 -0
- data/script/console +28 -0
- data/script/start_rabbit +29 -0
- data/test/beetle/deduplication_store_test.rb +3 -1
- metadata +25 -4
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2010 XING AG
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@@ -0,0 +1,124 @@
|
|
1
|
+
= Automatic Redis Failover for Beetle
|
2
|
+
|
3
|
+
== Introduction
|
4
|
+
|
5
|
+
Redis is used as the persistence layer in the AMQP message deduplication
|
6
|
+
process. Because it is such a critical piece in our infrastructure, it is
|
7
|
+
essential that a failure of this service is as unlikely as possible. As our
|
8
|
+
AMQP workers are working in a highly distributed manner, all accessing the same
|
9
|
+
Redis server, a automatic failover to another Redis server has to be very
|
10
|
+
defensive and ensure that every worker in the system will switch to the new
|
11
|
+
server at the same time. If the new server would not get accepted from every
|
12
|
+
worker, a switch would not be possible. This ensures that even in the case of a
|
13
|
+
partitioned network it is impossible that two different workers use two
|
14
|
+
different Redis servers for message deduplication.
|
15
|
+
|
16
|
+
== Our goals
|
17
|
+
|
18
|
+
* opt-in, no need to use the redis-failover solution
|
19
|
+
* no single point of failure
|
20
|
+
* automatic switch in case of redis-master failure
|
21
|
+
* switch should not cause inconsistent data on the redis servers
|
22
|
+
* workers should be able to determine the current redis-master without asking
|
23
|
+
another process (as long as the redis servers are working)
|
24
|
+
|
25
|
+
== How it works
|
26
|
+
|
27
|
+
To ensure consistency, a service (the Redis Configuration Server - RCS) is
|
28
|
+
constantly checking the availability and configuration of the currently
|
29
|
+
configured Redis master server. If this service detects that the Redis master
|
30
|
+
is no longer available, it tries to find an alternative server (one of the
|
31
|
+
slaves) which could be promoted to be the new Redis master.
|
32
|
+
|
33
|
+
On every worker server runs another daemon, the Redis Configuration Client
|
34
|
+
(RCC) which listens to messages sent by the RCS.
|
35
|
+
|
36
|
+
If the RCS finds another potential Redis Master, it sends out a message to see
|
37
|
+
if all known RCCs are still available (once again to eliminate the risk of a
|
38
|
+
partitioned network) and if they agree to the master switch.
|
39
|
+
|
40
|
+
If all RCCs have answered to that message, the RCS sends out a message which
|
41
|
+
tells the RCCs to invalidate the current master.
|
42
|
+
|
43
|
+
This happens by deleting the contents of a special file which is used
|
44
|
+
by the workers to store the current Redis master (the content of that file is
|
45
|
+
the hostname:port of the currently active Redis master). By doing that, it is
|
46
|
+
ensured that no operations are done to the old Redis master server anymore, because the
|
47
|
+
AMQP workers check this file's mtime and reads its contents in case that the
|
48
|
+
file changed, before every Redis operation. When the file has been emptied, the
|
49
|
+
RCCs respond to the "invalidate" message of the RCS. When all RCCs have
|
50
|
+
responded, the RCS knows for sure that it is safe to switch the Redis master
|
51
|
+
now. It sends a "reconfigure" message with the new Redis master hostname:port
|
52
|
+
to the RCCs, which then write that value into their redis master file.
|
53
|
+
|
54
|
+
Additionally, the RCS sends reconfigure messages with the current Redis master
|
55
|
+
periodically, to allow new RCCs to pick up the current master. Plus it turns
|
56
|
+
all other redis servers into slaves of the current master.
|
57
|
+
|
58
|
+
=== Prerequisites
|
59
|
+
|
60
|
+
* one redis-configuration-server process ("RCS", on one server), one redis-configuration-client process ("RCC") on every worker server
|
61
|
+
* the RCS knows about all possible RCCs using a list of client ids
|
62
|
+
* the RCS and RCCs exchange messages via a "system queue"
|
63
|
+
|
64
|
+
=== Flow of actions
|
65
|
+
|
66
|
+
* on startup, an RCC can consult its redis master file to determine the current master without the help of the RCS by checking that it's still a master (or wait for the periodic reconfigure message with the current master from the RCS)
|
67
|
+
* when the RCS finds the master to be down, it will retry a couple of times before starting a reconfiguration round
|
68
|
+
* the RCS sends all RCCs a "ping" message to check if every client is there and able to to answer
|
69
|
+
* the RCCs acknowledge via a "pong" message if they can confirm the current master to be unavailable
|
70
|
+
* the RCS waits for *all* RCCs to reply via pong
|
71
|
+
* the RCS tells all RCCs to stop using the master by sending an "invalidate" message
|
72
|
+
* the RCCs acknowledge via an "invalidated" message if they can still confirm the current master to be unavailable
|
73
|
+
* the RCS waits for *all* RCCs to acknowledge the invalidation
|
74
|
+
* the RCS promotes the former slave to become the new master (by sending SLAVEOF no one)
|
75
|
+
* the RCS sends a "reconfigure" message containing the new master to every RCC
|
76
|
+
* the RCCs write the new master to their redis master file
|
77
|
+
|
78
|
+
=== Configuration
|
79
|
+
|
80
|
+
See Beetle::Configuration for setting redis configuration server and client options.
|
81
|
+
|
82
|
+
Please note:
|
83
|
+
Beetle::Configuration#redis_server must be a file path (not a redis host:port string) to use the redis failover. The RCS and RCCs store the current redis master in that file, and the handlers read from it.
|
84
|
+
|
85
|
+
== How to use it
|
86
|
+
|
87
|
+
This example uses two worker servers, identified by rcc-1 and rcc-2.
|
88
|
+
|
89
|
+
Please note:
|
90
|
+
All command line options can also be given as a yaml configuration file via the --config-file option.
|
91
|
+
|
92
|
+
=== On one server
|
93
|
+
|
94
|
+
Start the Redis Configuration Server:
|
95
|
+
|
96
|
+
beetle configuration_server start -- --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
|
97
|
+
|
98
|
+
Get help for starting/stopping the server:
|
99
|
+
|
100
|
+
beetle configuration_server -h
|
101
|
+
|
102
|
+
Get help for server options:
|
103
|
+
|
104
|
+
beetle configuration_server start -- -h
|
105
|
+
|
106
|
+
=== On every worker server
|
107
|
+
|
108
|
+
Start the Redis Configuration Client:
|
109
|
+
|
110
|
+
On first worker server:
|
111
|
+
|
112
|
+
beetle configuration_client start -- --client-id rcc-1
|
113
|
+
|
114
|
+
On second worker server:
|
115
|
+
|
116
|
+
beetle configuration_client start -- --client-id rcc-2
|
117
|
+
|
118
|
+
Get help for starting/stopping the client:
|
119
|
+
|
120
|
+
beetle configuration_client -h
|
121
|
+
|
122
|
+
Get help for client options:
|
123
|
+
|
124
|
+
beetle configuration_client start -- -h
|
data/RELEASE_NOTES.rdoc
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
= Release Notes
|
2
|
+
|
3
|
+
== Version 0.2.5
|
4
|
+
|
5
|
+
Added missing files to gem and rdoc
|
6
|
+
|
7
|
+
== Version 0.2.4
|
8
|
+
|
9
|
+
Log and send a system notification when pong message from unknown client received.
|
10
|
+
|
11
|
+
== Version 0.2.2
|
12
|
+
|
13
|
+
Patch release which upgrades to redis-rb 2.0.4. This enables us to drop our redis monkey
|
14
|
+
patch which enabled connection timeouts for earlier redis versions. Note that earlier
|
15
|
+
Beetle versions are not compatible with redis 2.0.4.
|
16
|
+
|
17
|
+
== Version 0.2.1
|
18
|
+
|
19
|
+
Improved error message when no rabbitmq broker is available.
|
20
|
+
|
21
|
+
== Version 0.2
|
22
|
+
|
23
|
+
This version adds support for automatic redis deduplication store failover (see separate
|
24
|
+
file REDIS_AUTO_FAILOVER.rdoc).
|
25
|
+
|
26
|
+
=== User visible changes
|
27
|
+
|
28
|
+
* it's possible to register auto deleted queues and exchanges
|
29
|
+
* Beetle::Client#configure returns self in order to simplify client setup
|
30
|
+
* it's possible to trace specific messages (see Beetle::Client#trace)
|
31
|
+
* default message handler timeout is 10 minutes now
|
32
|
+
* system wide configuration values can be specified via a yml formatted configuration
|
33
|
+
file (Beetle::Configuration#config_file)
|
34
|
+
* the config value redis_server specifies either a single server or a file path (used
|
35
|
+
by the automatic redis failover logic)
|
36
|
+
|
37
|
+
=== Fugs Bixed
|
38
|
+
|
39
|
+
* handle active_support seconds notation for handler timeouts correctly
|
40
|
+
* error handler was erroneously called for expired messages
|
41
|
+
* subscribers would block when some non beetle process posts an undecodable message
|
42
|
+
|
43
|
+
=== Gem Dependency Changes
|
44
|
+
|
45
|
+
* redis needs to be at least version 2.0.3
|
46
|
+
* we make use of the SystemTimer gem for ruby 1.8.7
|
47
|
+
|
48
|
+
== Version 0.1
|
49
|
+
|
50
|
+
Initial Release
|
data/Rakefile
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rcov/rcovtask'
|
4
|
+
require 'cucumber/rake/task'
|
5
|
+
|
6
|
+
# 1.8/1.9 compatible way of loading lib/beetle.rb
|
7
|
+
$:.unshift 'lib'
|
8
|
+
require 'beetle'
|
9
|
+
|
10
|
+
namespace :test do
|
11
|
+
namespace :coverage do
|
12
|
+
desc "Delete aggregate coverage data."
|
13
|
+
task(:clean) { rm_f "coverage.data" }
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Aggregate code coverage'
|
17
|
+
task :coverage => "test:coverage:clean"
|
18
|
+
|
19
|
+
Rcov::RcovTask.new(:coverage) do |t|
|
20
|
+
t.libs << "test"
|
21
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
22
|
+
t.output_dir = "test/coverage"
|
23
|
+
t.verbose = true
|
24
|
+
t.rcov_opts << "--exclude '.*' --include-file 'lib/beetle/'"
|
25
|
+
end
|
26
|
+
task :coverage do
|
27
|
+
system 'open test/coverage/index.html'
|
28
|
+
end if RUBY_PLATFORM =~ /darwin/
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
namespace :beetle do
|
33
|
+
task :test do
|
34
|
+
Beetle::Client.new.test
|
35
|
+
end
|
36
|
+
|
37
|
+
task :trace do
|
38
|
+
trap('INT'){ EM.stop_event_loop }
|
39
|
+
Beetle::Client.new.trace
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
namespace :rabbit do
|
44
|
+
def start(node_name, port)
|
45
|
+
script = File.expand_path(File.dirname(__FILE__)+"/script/start_rabbit")
|
46
|
+
puts "starting rabbit #{node_name} on port #{port}"
|
47
|
+
puts "type ^C a RETURN to abort"
|
48
|
+
sleep 1
|
49
|
+
exec "sudo #{script} #{node_name} #{port}"
|
50
|
+
end
|
51
|
+
desc "start rabbit instance 1"
|
52
|
+
task :start1 do
|
53
|
+
start "rabbit1", 5672
|
54
|
+
end
|
55
|
+
desc "start rabbit instance 2"
|
56
|
+
task :start2 do
|
57
|
+
start "rabbit2", 5673
|
58
|
+
end
|
59
|
+
desc "reset rabbit instances (deletes all data!)"
|
60
|
+
task :reset do
|
61
|
+
["rabbit1", "rabbit2"].each do |node|
|
62
|
+
`sudo rabbitmqctl -n #{node} stop_app`
|
63
|
+
`sudo rabbitmqctl -n #{node} reset`
|
64
|
+
`sudo rabbitmqctl -n #{node} start_app`
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
namespace :redis do
|
70
|
+
def config_file(suffix)
|
71
|
+
File.expand_path(File.dirname(__FILE__)+"/etc/redis-#{suffix}.conf")
|
72
|
+
end
|
73
|
+
desc "start main redis"
|
74
|
+
task :start1 do
|
75
|
+
exec "redis-server #{config_file(:master)}"
|
76
|
+
end
|
77
|
+
desc "start slave redis"
|
78
|
+
task :start2 do
|
79
|
+
exec "redis-server #{config_file(:slave)}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
Cucumber::Rake::Task.new(:cucumber) do |t|
|
84
|
+
t.cucumber_opts = "features --format progress"
|
85
|
+
end
|
86
|
+
|
87
|
+
task :default do
|
88
|
+
Rake::Task[:test].invoke
|
89
|
+
Rake::Task[:cucumber].invoke
|
90
|
+
end
|
91
|
+
|
92
|
+
Rake::TestTask.new do |t|
|
93
|
+
t.libs << "test"
|
94
|
+
t.test_files = FileList['test/**/*_test.rb']
|
95
|
+
t.verbose = true
|
96
|
+
end
|
97
|
+
|
98
|
+
require 'rake/rdoctask'
|
99
|
+
|
100
|
+
Rake::RDocTask.new do |rdoc|
|
101
|
+
rdoc.rdoc_dir = 'site/rdoc'
|
102
|
+
rdoc.title = 'Beetle'
|
103
|
+
rdoc.main = 'README.rdoc'
|
104
|
+
rdoc.options << '--line-numbers' << '--inline-source' << '--quiet'
|
105
|
+
rdoc.rdoc_files.include('**/*.rdoc')
|
106
|
+
rdoc.rdoc_files.include('MIT-LICENSE')
|
107
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
108
|
+
end
|
109
|
+
|
110
|
+
desc "build the beetle gem"
|
111
|
+
task :build do
|
112
|
+
system("gem build beetle.gemspec")
|
113
|
+
end
|
data/beetle.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = "beetle"
|
3
|
-
s.version = "0.2.
|
3
|
+
s.version = "0.2.5"
|
4
4
|
|
5
5
|
s.required_rubygems_version = ">= 1.3.1"
|
6
6
|
s.authors = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
|
@@ -10,8 +10,8 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.summary = "High Availability AMQP Messaging with Redundant Queues"
|
11
11
|
s.email = "developers@xing.com"
|
12
12
|
s.executables = ["beetle"]
|
13
|
-
s.extra_rdoc_files = [
|
14
|
-
s.files = Dir['{examples,ext,lib}/**/*.rb'] + %w(beetle.gemspec
|
13
|
+
s.extra_rdoc_files = Dir['**/*.rdoc'] + %w(MIT-LICENSE)
|
14
|
+
s.files = Dir['{examples,ext,lib}/**/*.rb'] + Dir['{features,script}/**/*'] + %w(beetle.gemspec Rakefile)
|
15
15
|
s.extensions = 'ext/mkrf_conf.rb'
|
16
16
|
s.homepage = "http://xing.github.com/beetle/"
|
17
17
|
s.rdoc_options = ["--charset=UTF-8"]
|
@@ -0,0 +1,23 @@
|
|
1
|
+
=== Cucumber
|
2
|
+
|
3
|
+
Beetle ships with a cucumber feature to test the automatic redis failover
|
4
|
+
as an integration test.
|
5
|
+
|
6
|
+
To run it, you have to start a RabbitMQ.
|
7
|
+
|
8
|
+
The top level Rakefile comes with targets to start several RabbitMQ instances locally.
|
9
|
+
Make sure the corresponding binaries are in your search path. Open a new shell
|
10
|
+
and execute the following command:
|
11
|
+
|
12
|
+
rake rabbit:start1
|
13
|
+
|
14
|
+
Then you can run the cucumber feature by running:
|
15
|
+
|
16
|
+
cucumber
|
17
|
+
|
18
|
+
or
|
19
|
+
|
20
|
+
rake cucumber
|
21
|
+
|
22
|
+
|
23
|
+
Note: Cucumber will automatically run after the unit test when you run rake.
|
@@ -0,0 +1,105 @@
|
|
1
|
+
Feature: Redis auto failover
|
2
|
+
In order to eliminate a single point of failure
|
3
|
+
Beetle handlers should automatically switch to a new redis master in case of a redis master failure
|
4
|
+
|
5
|
+
Background:
|
6
|
+
Given a redis server "redis-1" exists as master
|
7
|
+
And a redis server "redis-2" exists as slave of "redis-1"
|
8
|
+
|
9
|
+
Scenario: Successful redis master switch
|
10
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
11
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
12
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
13
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
14
|
+
And redis server "redis-1" is down
|
15
|
+
And the retry timeout for the redis master check is reached
|
16
|
+
Then a system notification for "redis-1" not being available should be sent
|
17
|
+
And the role of redis server "redis-2" should be "master"
|
18
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
19
|
+
And the redis master of "rc-client-2" should be "redis-2"
|
20
|
+
And the redis master of the beetle handler should be "redis-2"
|
21
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
22
|
+
Given a redis server "redis-1" exists as master
|
23
|
+
Then the role of redis server "redis-1" should be "slave"
|
24
|
+
|
25
|
+
Scenario: Redis master only temporarily down (no switch necessary)
|
26
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
27
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
28
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
29
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
30
|
+
And redis server "redis-1" is down for less seconds than the retry timeout for the redis master check
|
31
|
+
And the retry timeout for the redis master check is reached
|
32
|
+
Then the role of redis server "redis-1" should be "master"
|
33
|
+
Then the role of redis server "redis-2" should be "slave"
|
34
|
+
And the redis master of "rc-client-1" should be "redis-1"
|
35
|
+
And the redis master of "rc-client-2" should be "redis-1"
|
36
|
+
And the redis master of the beetle handler should be "redis-1"
|
37
|
+
|
38
|
+
Scenario: Not all redis configuration clients available (no switch possible)
|
39
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
40
|
+
And redis server "redis-1" is down
|
41
|
+
And the retry timeout for the redis master check is reached
|
42
|
+
Then the role of redis server "redis-2" should be "slave"
|
43
|
+
|
44
|
+
Scenario: No redis slave available to become new master (no switch possible)
|
45
|
+
Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
|
46
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
47
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
|
48
|
+
And redis server "redis-1" is down
|
49
|
+
And redis server "redis-2" is down
|
50
|
+
And the retry timeout for the redis master check is reached
|
51
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
52
|
+
And the redis master of "rc-client-2" should be "redis-1"
|
53
|
+
And a system notification for no slave available to become new master should be sent
|
54
|
+
|
55
|
+
Scenario: Redis configuration client starts while no redis master available
|
56
|
+
Given redis server "redis-1" is down
|
57
|
+
And redis server "redis-2" is down
|
58
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
59
|
+
And the retry timeout for the redis master determination is reached
|
60
|
+
Then the redis master of "rc-client-1" should be undefined
|
61
|
+
|
62
|
+
Scenario: Redis configuration client starts while no redis master available but master file exists
|
63
|
+
Given redis server "redis-1" is down
|
64
|
+
And redis server "redis-2" is down
|
65
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
66
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
67
|
+
And the retry timeout for the redis master determination is reached
|
68
|
+
Then the redis master of "rc-client-1" should be undefined
|
69
|
+
|
70
|
+
Scenario: Redis configuration client starts while both redis servers are master
|
71
|
+
Given redis server "redis-2" is master
|
72
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
73
|
+
Then the redis master of "rc-client-1" should be undefined
|
74
|
+
|
75
|
+
Scenario: Redis configuration client starts while both redis servers are master but master file exists
|
76
|
+
Given redis server "redis-2" is master
|
77
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
78
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
79
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
80
|
+
|
81
|
+
Scenario: Redis configuration client starts while both redis servers are slave
|
82
|
+
Given a redis server "redis-3" exists as master
|
83
|
+
And redis server "redis-1" is slave of "redis-3"
|
84
|
+
And redis server "redis-2" is slave of "redis-3"
|
85
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
86
|
+
Then the redis master of "rc-client-1" should be undefined
|
87
|
+
|
88
|
+
Scenario: Redis configuration client starts while both redis servers are slave but master file exists
|
89
|
+
Given a redis server "redis-3" exists as master
|
90
|
+
And redis server "redis-1" is slave of "redis-3"
|
91
|
+
And redis server "redis-2" is slave of "redis-3"
|
92
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
93
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
94
|
+
Then the redis master of "rc-client-1" should be undefined
|
95
|
+
|
96
|
+
Scenario: Redis configuration client starts while there is a redis master but no slave
|
97
|
+
Given redis server "redis-2" is down
|
98
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
99
|
+
Then the redis master of "rc-client-1" should be undefined
|
100
|
+
|
101
|
+
Scenario: Redis configuration client starts while there is a redis master but no slave but master file exists
|
102
|
+
Given redis server "redis-2" is down
|
103
|
+
And an old redis master file for "rc-client-1" with master "redis-1" exists
|
104
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
|
105
|
+
Then the redis master of "rc-client-1" should be "redis-1"
|
@@ -0,0 +1,133 @@
|
|
1
|
+
Given /^a redis server "([^\"]*)" exists as master$/ do |redis_name|
|
2
|
+
TestDaemons::Redis[redis_name].start
|
3
|
+
TestDaemons::Redis[redis_name].master
|
4
|
+
end
|
5
|
+
|
6
|
+
Given /^a redis server "([^\"]*)" exists as slave of "([^\"]*)"$/ do |redis_name, redis_master_name|
|
7
|
+
TestDaemons::Redis[redis_name].start
|
8
|
+
Given "redis server \"#{redis_name}\" is slave of \"#{redis_master_name}\""
|
9
|
+
end
|
10
|
+
|
11
|
+
Given /^redis server "([^\"]*)" is master$/ do |redis_name|
|
12
|
+
TestDaemons::Redis[redis_name].master
|
13
|
+
end
|
14
|
+
|
15
|
+
Given /^redis server "([^\"]*)" is slave of "([^\"]*)"$/ do |redis_name, redis_master_name|
|
16
|
+
TestDaemons::Redis[redis_name].slave_of(TestDaemons::Redis[redis_master_name].port)
|
17
|
+
master = TestDaemons::Redis[redis_master_name].redis
|
18
|
+
slave = TestDaemons::Redis[redis_name].redis
|
19
|
+
begin
|
20
|
+
sleep 1
|
21
|
+
end while !slave.slave_of?(master.host, master.port)
|
22
|
+
end
|
23
|
+
|
24
|
+
Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" exists$/ do |redis_names, redis_configuration_client_names|
|
25
|
+
redis_servers = redis_names.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
|
26
|
+
TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names)
|
27
|
+
end
|
28
|
+
|
29
|
+
Given /^a redis configuration client "([^\"]*)" using redis servers "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_names|
|
30
|
+
redis_servers = redis_names.split(",").map do |redis_name|
|
31
|
+
TestDaemons::Redis[redis_name].ip_with_port
|
32
|
+
end
|
33
|
+
TestDaemons::RedisConfigurationClient[redis_configuration_client_name].start
|
34
|
+
end
|
35
|
+
|
36
|
+
Given /^redis server "([^\"]*)" is down$/ do |redis_name|
|
37
|
+
TestDaemons::Redis[redis_name].stop
|
38
|
+
end
|
39
|
+
|
40
|
+
Given /^the retry timeout for the redis master check is reached$/ do
|
41
|
+
basetime = Time.now
|
42
|
+
i = 0
|
43
|
+
while (i <= 10.0) do
|
44
|
+
break if TestDaemons::RedisConfigurationClient.instances.values.all? {|instance| File.mtime(instance.redis_master_file) > basetime rescue false}
|
45
|
+
i += 0.1
|
46
|
+
sleep(0.1)
|
47
|
+
end
|
48
|
+
sleep 1 # give it time to switch because the modified mtime might be because of the initial invalidation and not the switch
|
49
|
+
end
|
50
|
+
|
51
|
+
Given /^a beetle handler using the redis-master file from "([^\"]*)" exists$/ do |redis_configuration_client_name|
|
52
|
+
master_file = redis_master_file(redis_configuration_client_name)
|
53
|
+
`ruby features/support/beetle_handler start -- --redis-master-file=#{master_file}`
|
54
|
+
assert File.exist?(master_file), "file #{master_file} does not exist"
|
55
|
+
end
|
56
|
+
|
57
|
+
Given /^redis server "([^\"]*)" is down for less seconds than the retry timeout for the redis master check$/ do |redis_name|
|
58
|
+
TestDaemons::Redis[redis_name].restart(1)
|
59
|
+
end
|
60
|
+
|
61
|
+
Given /^the retry timeout for the redis master determination is reached$/ do
|
62
|
+
sleep 1
|
63
|
+
end
|
64
|
+
|
65
|
+
Given /^redis server "([^\"]*)" is coming back$/ do |redis_name|
|
66
|
+
TestDaemons::Redis[redis_name].start
|
67
|
+
end
|
68
|
+
|
69
|
+
Given /^an old redis master file for "([^\"]*)" with master "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_name|
|
70
|
+
master_file = redis_master_file(redis_configuration_client_name)
|
71
|
+
File.open(master_file, 'w') do |f|
|
72
|
+
f.puts TestDaemons::Redis[redis_name].ip_with_port
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redis_name, role|
|
78
|
+
expected_role = false
|
79
|
+
10.times do
|
80
|
+
expected_role = true and break if TestDaemons::Redis[redis_name].__send__ "#{role}?"
|
81
|
+
sleep 1
|
82
|
+
end
|
83
|
+
assert expected_role, "#{redis_name} is not a #{role}"
|
84
|
+
end
|
85
|
+
|
86
|
+
Then /^the redis master of "([^\"]*)" should be "([^\"]*)"$/ do |redis_configuration_client_name, redis_name|
|
87
|
+
master_file = redis_master_file(redis_configuration_client_name)
|
88
|
+
master = false
|
89
|
+
server_info = nil
|
90
|
+
10.times do
|
91
|
+
server_info = File.read(master_file).chomp if File.exist?(master_file)
|
92
|
+
master = true and break if TestDaemons::Redis[redis_name].ip_with_port == server_info
|
93
|
+
sleep 1
|
94
|
+
end
|
95
|
+
assert master, "#{redis_name} is not master of #{redis_configuration_client_name}, master file content: #{server_info.inspect}"
|
96
|
+
end
|
97
|
+
|
98
|
+
Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configuration_client_name|
|
99
|
+
master_file = redis_master_file(redis_configuration_client_name)
|
100
|
+
empty = false
|
101
|
+
server_info = nil
|
102
|
+
10.times do
|
103
|
+
server_info = File.read(master_file).chomp if File.exist?(master_file)
|
104
|
+
empty = server_info == ""
|
105
|
+
break if empty
|
106
|
+
sleep 1
|
107
|
+
end
|
108
|
+
assert empty, "master file is not empty: #{server_info}"
|
109
|
+
end
|
110
|
+
|
111
|
+
Then /^the redis master of the beetle handler should be "([^\"]*)"$/ do |redis_name|
|
112
|
+
Beetle.config.servers = "localhost:5672" # rabbitmq
|
113
|
+
Beetle.config.logger.level = Logger::INFO
|
114
|
+
client = Beetle::Client.new
|
115
|
+
client.register_queue(:echo)
|
116
|
+
client.register_message(:echo)
|
117
|
+
assert_equal TestDaemons::Redis[redis_name].ip_with_port, client.rpc(:echo, 'nil').second
|
118
|
+
end
|
119
|
+
|
120
|
+
Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
|
121
|
+
text = "Redis master '#{TestDaemons::Redis[redis_name].ip_with_port}' not available"
|
122
|
+
assert_match /#{text}/, File.readlines(system_notification_log_path).last
|
123
|
+
end
|
124
|
+
|
125
|
+
Then /^a system notification for switching from "([^\"]*)" to "([^\"]*)" should be sent$/ do |old_redis_master_name, new_redis_master_name|
|
126
|
+
text = "Setting redis master to '#{TestDaemons::Redis[new_redis_master_name].ip_with_port}' (was '#{TestDaemons::Redis[old_redis_master_name].ip_with_port}')"
|
127
|
+
assert_match /#{Regexp.escape(text)}/, File.read(system_notification_log_path)
|
128
|
+
end
|
129
|
+
|
130
|
+
Then /^a system notification for no slave available to become new master should be sent$/ do
|
131
|
+
text = "Redis master could not be switched, no slave available to become new master"
|
132
|
+
assert_match /#{text}/, File.readlines(system_notification_log_path).last
|
133
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
|
3
|
+
require "rubygems"
|
4
|
+
require "daemons"
|
5
|
+
require "optparse"
|
6
|
+
require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
|
7
|
+
|
8
|
+
tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
|
9
|
+
|
10
|
+
Daemons.run_proc("beetle_handler", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
|
11
|
+
opts = OptionParser.new
|
12
|
+
|
13
|
+
opts.on("-f", "--redis-master-file path", String) do |val|
|
14
|
+
Beetle.config.redis_server = val
|
15
|
+
end
|
16
|
+
|
17
|
+
opts.parse!(ARGV - ["start", "--"])
|
18
|
+
|
19
|
+
Beetle.config.servers = "localhost:5672" # rabbitmq
|
20
|
+
|
21
|
+
# set Beetle log level to info, less noisy than debug
|
22
|
+
Beetle.config.logger.level = Logger::INFO
|
23
|
+
|
24
|
+
client = Beetle::Client.new.configure :auto_delete => true do |config|
|
25
|
+
config.queue(:echo)
|
26
|
+
config.message(:echo)
|
27
|
+
config.handler(:echo) {|message| client.deduplication_store.redis.server rescue "no redis master"}
|
28
|
+
end
|
29
|
+
client.listen do
|
30
|
+
puts "Started beetle handler"
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require File.expand_path('../../../lib/beetle', __FILE__)
|
2
|
+
|
3
|
+
# Allow using Test::Unit for step assertions
|
4
|
+
# See http://wiki.github.com/aslakhellesoy/cucumber/using-testunit
|
5
|
+
require 'test/unit/assertions'
|
6
|
+
World(Test::Unit::Assertions)
|
7
|
+
|
8
|
+
Before do
|
9
|
+
`ruby features/support/system_notification_logger start`
|
10
|
+
end
|
11
|
+
|
12
|
+
After do
|
13
|
+
cleanup_test_env
|
14
|
+
end
|
15
|
+
|
16
|
+
at_exit do
|
17
|
+
cleanup_test_env
|
18
|
+
end
|
19
|
+
|
20
|
+
def cleanup_test_env
|
21
|
+
TestDaemons::RedisConfigurationClient.stop_all
|
22
|
+
TestDaemons::RedisConfigurationServer.stop
|
23
|
+
|
24
|
+
`ruby features/support/beetle_handler stop`
|
25
|
+
redis_master_files = tmp_path + "/redis-master-*"
|
26
|
+
`rm -f #{redis_master_files}`
|
27
|
+
|
28
|
+
`ruby features/support/system_notification_logger stop`
|
29
|
+
`rm -f #{system_notification_log_path}`
|
30
|
+
|
31
|
+
TestDaemons::Redis.stop_all
|
32
|
+
end
|
33
|
+
|
34
|
+
def redis_master_file(client_name)
|
35
|
+
tmp_path + "/redis-master-#{client_name}"
|
36
|
+
end
|
37
|
+
|
38
|
+
def first_redis_configuration_client_pid
|
39
|
+
File.read("redis_configuration_client0.pid").chomp.to_i
|
40
|
+
end
|
41
|
+
|
42
|
+
def system_notification_log_path
|
43
|
+
tmp_path + "/system_notifications.log"
|
44
|
+
end
|
45
|
+
|
46
|
+
def tmp_path
|
47
|
+
File.expand_path("../../../tmp", __FILE__)
|
48
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
|
3
|
+
require "rubygems"
|
4
|
+
require "daemons"
|
5
|
+
require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
|
6
|
+
|
7
|
+
tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
|
8
|
+
system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
|
9
|
+
|
10
|
+
Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
|
11
|
+
Beetle.config.servers = "localhost:5672, localhost:5673" # rabbitmq
|
12
|
+
|
13
|
+
# set Beetle log level to info, less noisy than debug
|
14
|
+
Beetle.config.logger.level = Logger::DEBUG
|
15
|
+
|
16
|
+
client = Beetle::Client.new
|
17
|
+
client.configure :exchange => :system, :auto_delete => true do |config|
|
18
|
+
config.message :system_notification
|
19
|
+
config.queue :system_notification
|
20
|
+
config.handler :system_notification do |message|
|
21
|
+
payload = ActiveSupport::JSON.decode(message.data)
|
22
|
+
text = payload["message"]
|
23
|
+
puts "Writing message to #{system_notification_log_file_path}: #{text}"
|
24
|
+
File.open(system_notification_log_file_path, "a+") do |f|
|
25
|
+
f << text
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
puts "Started system notification logger"
|
30
|
+
client.listen
|
31
|
+
end
|
@@ -0,0 +1,189 @@
|
|
1
|
+
# Redis configuration file example
|
2
|
+
|
3
|
+
# By default Redis does not run as a daemon. Use 'yes' if you need it.
|
4
|
+
# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
|
5
|
+
daemonize yes
|
6
|
+
|
7
|
+
# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
|
8
|
+
# You can specify a custom pid file location here.
|
9
|
+
pidfile <%= pid_file %>
|
10
|
+
|
11
|
+
# Accept connections on the specified port, default is 6379
|
12
|
+
port <%= port %>
|
13
|
+
|
14
|
+
# If you want you can bind a single interface, if the bind option is not
|
15
|
+
# specified all the interfaces will listen for connections.
|
16
|
+
#
|
17
|
+
# bind 127.0.0.1
|
18
|
+
|
19
|
+
# Close the connection after a client is idle for N seconds (0 to disable)
|
20
|
+
timeout 300
|
21
|
+
|
22
|
+
# Set server verbosity to 'debug'
|
23
|
+
# it can be one of:
|
24
|
+
# debug (a lot of information, useful for development/testing)
|
25
|
+
# notice (moderately verbose, what you want in production probably)
|
26
|
+
# warning (only very important / critical messages are logged)
|
27
|
+
loglevel debug
|
28
|
+
|
29
|
+
# Specify the log file name. Also 'stdout' can be used to force
|
30
|
+
# the demon to log on the standard output. Note that if you use standard
|
31
|
+
# output for logging but daemonize, logs will be sent to /dev/null
|
32
|
+
logfile <%= log_file %>
|
33
|
+
|
34
|
+
# Set the number of databases. The default database is DB 0, you can select
|
35
|
+
# a different one on a per-connection basis using SELECT <dbid> where
|
36
|
+
# dbid is a number between 0 and 'databases'-1
|
37
|
+
databases 16
|
38
|
+
|
39
|
+
################################ SNAPSHOTTING #################################
|
40
|
+
#
|
41
|
+
# Save the DB on disk:
|
42
|
+
#
|
43
|
+
# save <seconds> <changes>
|
44
|
+
#
|
45
|
+
# Will save the DB if both the given number of seconds and the given
|
46
|
+
# number of write operations against the DB occurred.
|
47
|
+
#
|
48
|
+
# In the example below the behaviour will be to save:
|
49
|
+
# after 900 sec (15 min) if at least 1 key changed
|
50
|
+
# after 300 sec (5 min) if at least 10 keys changed
|
51
|
+
# after 60 sec if at least 10000 keys changed
|
52
|
+
save 900 1
|
53
|
+
save 300 10
|
54
|
+
save 60 10000
|
55
|
+
|
56
|
+
# Compress string objects using LZF when dump .rdb databases?
|
57
|
+
# For default that's set to 'yes' as it's almost always a win.
|
58
|
+
# If you want to save some CPU in the saving child set it to 'no' but
|
59
|
+
# the dataset will likely be bigger if you have compressible values or keys.
|
60
|
+
rdbcompression yes
|
61
|
+
|
62
|
+
# The filename where to dump the DB
|
63
|
+
dbfilename dump.rdb
|
64
|
+
|
65
|
+
# For default save/load DB in/from the working directory
|
66
|
+
# Note that you must specify a directory not a file name.
|
67
|
+
dir <%= dir %>
|
68
|
+
|
69
|
+
################################# REPLICATION #################################
|
70
|
+
|
71
|
+
# Master-Slave replication. Use slaveof to make a Redis instance a copy of
|
72
|
+
# another Redis server. Note that the configuration is local to the slave
|
73
|
+
# so for example it is possible to configure the slave to save the DB with a
|
74
|
+
# different interval, or to listen to another port, and so on.
|
75
|
+
#
|
76
|
+
# slaveof <masterip> <masterport>
|
77
|
+
|
78
|
+
# If the master is password protected (using the "requirepass" configuration
|
79
|
+
# directive below) it is possible to tell the slave to authenticate before
|
80
|
+
# starting the replication synchronization process, otherwise the master will
|
81
|
+
# refuse the slave request.
|
82
|
+
#
|
83
|
+
# masterauth <master-password>
|
84
|
+
|
85
|
+
################################## SECURITY ###################################
|
86
|
+
|
87
|
+
# Require clients to issue AUTH <PASSWORD> before processing any other
|
88
|
+
# commands. This might be useful in environments in which you do not trust
|
89
|
+
# others with access to the host running redis-server.
|
90
|
+
#
|
91
|
+
# This should stay commented out for backward compatibility and because most
|
92
|
+
# people do not need auth (e.g. they run their own servers).
|
93
|
+
#
|
94
|
+
# requirepass foobared
|
95
|
+
|
96
|
+
################################### LIMITS ####################################
|
97
|
+
|
98
|
+
# Set the max number of connected clients at the same time. By default there
|
99
|
+
# is no limit, and it's up to the number of file descriptors the Redis process
|
100
|
+
# is able to open. The special value '0' means no limts.
|
101
|
+
# Once the limit is reached Redis will close all the new connections sending
|
102
|
+
# an error 'max number of clients reached'.
|
103
|
+
#
|
104
|
+
# maxclients 128
|
105
|
+
|
106
|
+
# Don't use more memory than the specified amount of bytes.
|
107
|
+
# When the memory limit is reached Redis will try to remove keys with an
|
108
|
+
# EXPIRE set. It will try to start freeing keys that are going to expire
|
109
|
+
# in little time and preserve keys with a longer time to live.
|
110
|
+
# Redis will also try to remove objects from free lists if possible.
|
111
|
+
#
|
112
|
+
# If all this fails, Redis will start to reply with errors to commands
|
113
|
+
# that will use more memory, like SET, LPUSH, and so on, and will continue
|
114
|
+
# to reply to most read-only commands like GET.
|
115
|
+
#
|
116
|
+
# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
|
117
|
+
# 'state' server or cache, not as a real DB. When Redis is used as a real
|
118
|
+
# database the memory usage will grow over the weeks, it will be obvious if
|
119
|
+
# it is going to use too much memory in the long run, and you'll have the time
|
120
|
+
# to upgrade. With maxmemory after the limit is reached you'll start to get
|
121
|
+
# errors for write operations, and this may even lead to DB inconsistency.
|
122
|
+
#
|
123
|
+
# maxmemory <bytes>
|
124
|
+
|
125
|
+
############################## APPEND ONLY MODE ###############################
|
126
|
+
|
127
|
+
# By default Redis asynchronously dumps the dataset on disk. If you can live
|
128
|
+
# with the idea that the latest records will be lost if something like a crash
|
129
|
+
# happens this is the preferred way to run Redis. If instead you care a lot
|
130
|
+
# about your data and don't want to that a single record can get lost you should
|
131
|
+
# enable the append only mode: when this mode is enabled Redis will append
|
132
|
+
# every write operation received in the file appendonly.log. This file will
|
133
|
+
# be read on startup in order to rebuild the full dataset in memory.
|
134
|
+
#
|
135
|
+
# Note that you can have both the async dumps and the append only file if you
|
136
|
+
# like (you have to comment the "save" statements above to disable the dumps).
|
137
|
+
# Still if append only mode is enabled Redis will load the data from the
|
138
|
+
# log file at startup ignoring the dump.rdb file.
|
139
|
+
#
|
140
|
+
# The name of the append only file is "appendonly.log"
|
141
|
+
#
|
142
|
+
# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
|
143
|
+
# log file in background when it gets too big.
|
144
|
+
|
145
|
+
appendonly yes
|
146
|
+
|
147
|
+
# The fsync() call tells the Operating System to actually write data on disk
|
148
|
+
# instead to wait for more data in the output buffer. Some OS will really flush
|
149
|
+
# data on disk, some other OS will just try to do it ASAP.
|
150
|
+
#
|
151
|
+
# Redis supports three different modes:
|
152
|
+
#
|
153
|
+
# no: don't fsync, just let the OS flush the data when it wants. Faster.
|
154
|
+
# always: fsync after every write to the append only log . Slow, Safest.
|
155
|
+
# everysec: fsync only if one second passed since the last fsync. Compromise.
|
156
|
+
#
|
157
|
+
# The default is "always" that's the safer of the options. It's up to you to
|
158
|
+
# understand if you can relax this to "everysec" that will fsync every second
|
159
|
+
# or to "no" that will let the operating system flush the output buffer when
|
160
|
+
# it want, for better performances (but if you can live with the idea of
|
161
|
+
# some data loss consider the default persistence mode that's snapshotting).
|
162
|
+
|
163
|
+
# appendfsync always
|
164
|
+
appendfsync everysec
|
165
|
+
# appendfsync no
|
166
|
+
|
167
|
+
############################### ADVANCED CONFIG ###############################
|
168
|
+
|
169
|
+
# Glue small output buffers together in order to send small replies in a
|
170
|
+
# single TCP packet. Uses a bit more CPU but most of the times it is a win
|
171
|
+
# in terms of number of queries per second. Use 'yes' if unsure.
|
172
|
+
glueoutputbuf yes
|
173
|
+
|
174
|
+
# Use object sharing. Can save a lot of memory if you have many common
|
175
|
+
# string in your dataset, but performs lookups against the shared objects
|
176
|
+
# pool so it uses more CPU and can be a bit slower. Usually it's a good
|
177
|
+
# idea.
|
178
|
+
#
|
179
|
+
# When object sharing is enabled (shareobjects yes) you can use
|
180
|
+
# shareobjectspoolsize to control the size of the pool used in order to try
|
181
|
+
# object sharing. A bigger pool size will lead to better sharing capabilities.
|
182
|
+
# In general you want this value to be at least the double of the number of
|
183
|
+
# very common strings you have in your dataset.
|
184
|
+
#
|
185
|
+
# WARNING: object sharing is experimental, don't enable this feature
|
186
|
+
# in production before of Redis 1.0-stable. Still please try this feature in
|
187
|
+
# your development environment so that we can test it better.
|
188
|
+
shareobjects no
|
189
|
+
shareobjectspoolsize 1024
|
@@ -0,0 +1,186 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'erb'
|
4
|
+
require 'redis'
|
5
|
+
require 'lib/beetle/redis_ext'
|
6
|
+
require 'daemon_controller'
|
7
|
+
|
8
|
+
module TestDaemons
|
9
|
+
class Redis
|
10
|
+
|
11
|
+
@@instances = {}
|
12
|
+
@@next_available_port = 6381
|
13
|
+
|
14
|
+
attr_reader :name, :port
|
15
|
+
|
16
|
+
def initialize(name)
|
17
|
+
@name = name
|
18
|
+
@port = @@next_available_port
|
19
|
+
|
20
|
+
@@next_available_port += 1
|
21
|
+
@@instances[name] = self
|
22
|
+
end
|
23
|
+
|
24
|
+
class << self
|
25
|
+
def find_or_initialize_by_name(name)
|
26
|
+
@@instances[name] ||= new(name)
|
27
|
+
end
|
28
|
+
alias_method :[], :find_or_initialize_by_name
|
29
|
+
|
30
|
+
def stop_all
|
31
|
+
@@instances.values.each{|i| i.stop}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def start
|
36
|
+
create_dir
|
37
|
+
create_config
|
38
|
+
daemon_controller.start
|
39
|
+
end
|
40
|
+
|
41
|
+
def restart(delay=1)
|
42
|
+
redis.shutdown rescue Errno::ECONNREFUSED
|
43
|
+
sleep delay
|
44
|
+
`redis-server #{config_filename}`
|
45
|
+
end
|
46
|
+
|
47
|
+
def stop
|
48
|
+
# TODO: Might need to be moved into RedisConfigurationServer
|
49
|
+
# 10.times do
|
50
|
+
# break if (redis.info["bgsave_in_progress"]) == 0 rescue false
|
51
|
+
# sleep 1
|
52
|
+
# end
|
53
|
+
daemon_controller.stop
|
54
|
+
ensure
|
55
|
+
cleanup
|
56
|
+
end
|
57
|
+
|
58
|
+
def cleanup
|
59
|
+
remove_dir
|
60
|
+
remove_config
|
61
|
+
remove_pid_file
|
62
|
+
end
|
63
|
+
|
64
|
+
# TODO: The retry logic must be moved into RedisConfigurationServer
|
65
|
+
def master
|
66
|
+
tries = 0
|
67
|
+
begin
|
68
|
+
redis.master!
|
69
|
+
rescue Errno::ECONNREFUSED, Errno::EAGAIN
|
70
|
+
puts "master role setting for #{name} failed: #{$!}"
|
71
|
+
sleep 1
|
72
|
+
retry if (tries+=1) > 5
|
73
|
+
raise "could not setup master #{name} #{$!}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def running?
|
78
|
+
cmd = "ps aux | grep 'redis-server #{config_filename}' | grep -v grep"
|
79
|
+
res = `#{cmd}`
|
80
|
+
x = res.chomp.split("\n")
|
81
|
+
x.size == 1
|
82
|
+
end
|
83
|
+
|
84
|
+
def available?
|
85
|
+
redis.available?
|
86
|
+
end
|
87
|
+
|
88
|
+
def master?
|
89
|
+
redis.master?
|
90
|
+
end
|
91
|
+
|
92
|
+
def slave?
|
93
|
+
redis.slave?
|
94
|
+
end
|
95
|
+
|
96
|
+
# TODO: Move to redis_ext
|
97
|
+
def slave_of(master_port)
|
98
|
+
tries = 0
|
99
|
+
begin
|
100
|
+
redis.slave_of!("127.0.0.1", master_port)
|
101
|
+
rescue Errno::ECONNREFUSED, Errno::EAGAIN
|
102
|
+
puts "slave role setting for #{name} failed: #{$!}"
|
103
|
+
sleep 1
|
104
|
+
retry if (tries+=1) > 5
|
105
|
+
raise "could not setup slave #{name}: #{$!}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def ip_with_port
|
110
|
+
"127.0.0.1:#{port}"
|
111
|
+
end
|
112
|
+
|
113
|
+
def redis
|
114
|
+
@redis ||= ::Redis.new(:host => "127.0.0.1", :port => port)
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def create_dir
|
120
|
+
FileUtils.mkdir(dir) unless File.exists?(dir)
|
121
|
+
end
|
122
|
+
|
123
|
+
def remove_dir
|
124
|
+
FileUtils.rm_r(dir) if File.exists?(dir)
|
125
|
+
end
|
126
|
+
|
127
|
+
def create_config
|
128
|
+
File.open(config_filename, "w") do |file|
|
129
|
+
file.puts config_content
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def remove_config
|
134
|
+
FileUtils.rm(config_filename) if File.exists?(config_filename)
|
135
|
+
end
|
136
|
+
|
137
|
+
def remove_pid_file
|
138
|
+
FileUtils.rm(pid_file) if File.exists?(pid_file)
|
139
|
+
end
|
140
|
+
|
141
|
+
def tmp_path
|
142
|
+
File.expand_path(File.dirname(__FILE__) + "/../../../tmp")
|
143
|
+
end
|
144
|
+
|
145
|
+
def config_filename
|
146
|
+
tmp_path + "/redis-test-server-#{name}.conf"
|
147
|
+
end
|
148
|
+
|
149
|
+
def config_content
|
150
|
+
template = ERB.new(File.read(config_template_filename))
|
151
|
+
template.result(binding)
|
152
|
+
end
|
153
|
+
|
154
|
+
def config_template_filename
|
155
|
+
File.dirname(__FILE__) + "/redis.conf.erb"
|
156
|
+
end
|
157
|
+
|
158
|
+
def pid_file
|
159
|
+
tmp_path + "/redis-test-server-#{name}.pid"
|
160
|
+
end
|
161
|
+
|
162
|
+
def pid
|
163
|
+
File.read(pid_file).chomp.to_i
|
164
|
+
end
|
165
|
+
|
166
|
+
def log_file
|
167
|
+
tmp_path + "/redis-test-server-#{name}.log"
|
168
|
+
end
|
169
|
+
|
170
|
+
def dir
|
171
|
+
tmp_path + "/redis-test-server-#{name}/"
|
172
|
+
end
|
173
|
+
|
174
|
+
def daemon_controller
|
175
|
+
@daemon_controller ||= DaemonController.new(
|
176
|
+
:identifier => "Redis test server",
|
177
|
+
:start_command => "redis-server #{config_filename}",
|
178
|
+
:ping_command => lambda { running? && available? },
|
179
|
+
:pid_file => pid_file,
|
180
|
+
:log_file => log_file,
|
181
|
+
:start_timeout => 5
|
182
|
+
)
|
183
|
+
end
|
184
|
+
|
185
|
+
end
|
186
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'daemon_controller'
|
2
|
+
|
3
|
+
module TestDaemons
|
4
|
+
class RedisConfigurationClient
|
5
|
+
cattr_reader :instances
|
6
|
+
@@next_daemon_id = 0
|
7
|
+
@@instances = {}
|
8
|
+
|
9
|
+
def initialize(name)
|
10
|
+
@name = name
|
11
|
+
@daemon_id = @@next_daemon_id
|
12
|
+
|
13
|
+
@@next_daemon_id += 1
|
14
|
+
@@instances[name] = self
|
15
|
+
end
|
16
|
+
|
17
|
+
class << self
|
18
|
+
def find_or_initialize_by_name(name)
|
19
|
+
@@instances[name] ||= new(name)
|
20
|
+
end
|
21
|
+
alias_method :[], :find_or_initialize_by_name
|
22
|
+
|
23
|
+
def stop_all
|
24
|
+
@@instances.values.each{|i| i.stop}
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def start
|
29
|
+
daemon_controller.start
|
30
|
+
end
|
31
|
+
|
32
|
+
def stop
|
33
|
+
daemon_controller.stop
|
34
|
+
end
|
35
|
+
|
36
|
+
def daemon_controller
|
37
|
+
@daemon_controller ||= DaemonController.new(
|
38
|
+
:identifier => "Redis configuration test client",
|
39
|
+
:start_command => "ruby bin/beetle configuration_client start -- -v --redis-master-file #{redis_master_file} --id #{@name} --pid-dir #{tmp_path} --amqp-servers 127.0.0.1:5672",
|
40
|
+
:ping_command => lambda{ true },
|
41
|
+
:pid_file => pid_file,
|
42
|
+
:log_file => log_file,
|
43
|
+
:start_timeout => 5
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
47
|
+
def redis_master_file
|
48
|
+
"#{tmp_path}/redis-master-#{@name}"
|
49
|
+
end
|
50
|
+
|
51
|
+
def pid_file
|
52
|
+
"#{tmp_path}/redis_configuration_client#{@daemon_id}.pid"
|
53
|
+
end
|
54
|
+
|
55
|
+
def log_file
|
56
|
+
"#{tmp_path}/redis_configuration_client.output"
|
57
|
+
end
|
58
|
+
|
59
|
+
def tmp_path
|
60
|
+
File.expand_path(File.dirname(__FILE__) + "/../../../tmp")
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'daemon_controller'
|
2
|
+
|
3
|
+
module TestDaemons
|
4
|
+
class RedisConfigurationServer
|
5
|
+
|
6
|
+
# At the moment, we need only one, so we implement the methods
|
7
|
+
# as class methods
|
8
|
+
|
9
|
+
@@redis_servers = ""
|
10
|
+
@@redis_configuration_clients = ""
|
11
|
+
|
12
|
+
def self.start(redis_servers, redis_configuration_clients)
|
13
|
+
stop
|
14
|
+
@@redis_servers = redis_servers
|
15
|
+
@@redis_configuration_clients = redis_configuration_clients
|
16
|
+
daemon_controller.start
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.stop
|
20
|
+
daemon_controller.stop
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.daemon_controller
|
24
|
+
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
25
|
+
DaemonController.new(
|
26
|
+
:identifier => "Redis configuration test server",
|
27
|
+
:start_command => "ruby bin/beetle configuration_server start -- -v --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-retry-interval 1 --pid-dir #{tmp_path} --amqp-servers 127.0.0.1:5672",
|
28
|
+
:ping_command => lambda{ true },
|
29
|
+
:pid_file => pid_file,
|
30
|
+
:log_file => log_file,
|
31
|
+
:start_timeout => 5
|
32
|
+
)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.redis_master_file
|
36
|
+
"#{tmp_path}/redis-master-rc-server"
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.pid_file
|
40
|
+
"#{tmp_path}/redis_configuration_server.pid"
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.log_file
|
44
|
+
"#{tmp_path}/redis_configuration_server.output"
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.tmp_path
|
48
|
+
File.expand_path(File.dirname(__FILE__) + "/../../../tmp")
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
data/script/console
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
options = { }
|
6
|
+
OptionParser.new do |opt|
|
7
|
+
opt.banner = "Usage: console [options]"
|
8
|
+
opt.on("--debugger", 'Enable ruby-debugging for the console.') { |v| options[:debugger] = v }
|
9
|
+
opt.parse!(ARGV)
|
10
|
+
end
|
11
|
+
|
12
|
+
libs = " -r irb/completion"
|
13
|
+
libs << %( -r ubygems)
|
14
|
+
libs << %( -r #{File.expand_path("../../lib/beetle.rb",__FILE__)})
|
15
|
+
|
16
|
+
if options[:debugger]
|
17
|
+
begin
|
18
|
+
require 'ruby-debug'
|
19
|
+
libs << " -r ruby-debug"
|
20
|
+
puts "=> Debugger enabled"
|
21
|
+
rescue Exception
|
22
|
+
puts "You need to install ruby-debug to run the console in debugging mode. With gems, use 'gem install ruby-debug'"
|
23
|
+
exit
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
|
28
|
+
exec "#{irb} #{libs} --simple-prompt"
|
data/script/start_rabbit
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# export RABBITMQ_MNESIA_BASE=/var/lib/rabbitmq/mnesia2
|
4
|
+
# Defaults to /var/lib/rabbitmq/mnesia. Set this to the directory where Mnesia
|
5
|
+
# database files should be placed.
|
6
|
+
|
7
|
+
# export RABBITMQ_LOG_BASE
|
8
|
+
# Defaults to /var/log/rabbitmq. Log files generated by the server will be placed
|
9
|
+
# in this directory.
|
10
|
+
|
11
|
+
export RABBITMQ_NODENAME=$1
|
12
|
+
# Defaults to rabbit. This can be useful if you want to run more than one node
|
13
|
+
# per machine - RABBITMQ_NODENAME should be unique per erlang-node-and-machine
|
14
|
+
# combination. See clustering on a single machine guide at <http://www.rab-
|
15
|
+
# bitmq.com/clustering.html#single-machine> for details.
|
16
|
+
|
17
|
+
# RABBITMQ_NODE_IP_ADDRESS
|
18
|
+
# Defaults to 0.0.0.0. This can be changed if you only want to bind to one net-
|
19
|
+
# work interface.
|
20
|
+
|
21
|
+
export RABBITMQ_NODE_PORT=$2
|
22
|
+
# Defaults to 5672.
|
23
|
+
|
24
|
+
# RABBITMQ_CLUSTER_CONFIG_FILE
|
25
|
+
# Defaults to /etc/rabbitmq/rabbitmq_cluster.config. If this file is present it
|
26
|
+
# is used by the server to auto-configure a RabbitMQ cluster. See the clustering
|
27
|
+
# guide at <http://www.rabbitmq.com/clustering.html> for details.
|
28
|
+
|
29
|
+
rabbitmq-server
|
@@ -73,7 +73,9 @@ module Beetle
|
|
73
73
|
|
74
74
|
private
|
75
75
|
def redis_test_master_file(server_string)
|
76
|
-
|
76
|
+
tmp_dir = File.expand_path("../../../tmp", __FILE__)
|
77
|
+
Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir)
|
78
|
+
path = tmp_dir + "/redis-master-for-unit-tests"
|
77
79
|
File.open(path, "w"){|f| f.puts server_string}
|
78
80
|
path
|
79
81
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 29
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 5
|
10
|
+
version: 0.2.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Stefan Kaes
|
@@ -18,7 +18,7 @@ autorequire:
|
|
18
18
|
bindir: bin
|
19
19
|
cert_chain: []
|
20
20
|
|
21
|
-
date: 2010-08-
|
21
|
+
date: 2010-08-23 00:00:00 +02:00
|
22
22
|
default_executable: beetle
|
23
23
|
dependencies:
|
24
24
|
- !ruby/object:Gem::Dependency
|
@@ -182,7 +182,12 @@ executables:
|
|
182
182
|
extensions:
|
183
183
|
- ext/mkrf_conf.rb
|
184
184
|
extra_rdoc_files:
|
185
|
+
- examples/README.rdoc
|
186
|
+
- features/README.rdoc
|
185
187
|
- README.rdoc
|
188
|
+
- REDIS_AUTO_FAILOVER.rdoc
|
189
|
+
- RELEASE_NOTES.rdoc
|
190
|
+
- MIT-LICENSE
|
186
191
|
files:
|
187
192
|
- examples/attempts.rb
|
188
193
|
- examples/handler_class.rb
|
@@ -212,9 +217,25 @@ files:
|
|
212
217
|
- lib/beetle/redis_server_info.rb
|
213
218
|
- lib/beetle/subscriber.rb
|
214
219
|
- lib/beetle.rb
|
220
|
+
- features/README.rdoc
|
221
|
+
- features/redis_auto_failover.feature
|
222
|
+
- features/step_definitions/redis_auto_failover_steps.rb
|
223
|
+
- features/support/beetle_handler
|
224
|
+
- features/support/env.rb
|
225
|
+
- features/support/system_notification_logger
|
226
|
+
- features/support/test_daemons/redis.conf.erb
|
227
|
+
- features/support/test_daemons/redis.rb
|
228
|
+
- features/support/test_daemons/redis_configuration_client.rb
|
229
|
+
- features/support/test_daemons/redis_configuration_server.rb
|
230
|
+
- script/console
|
231
|
+
- script/start_rabbit
|
215
232
|
- beetle.gemspec
|
233
|
+
- Rakefile
|
216
234
|
- examples/README.rdoc
|
217
235
|
- README.rdoc
|
236
|
+
- REDIS_AUTO_FAILOVER.rdoc
|
237
|
+
- RELEASE_NOTES.rdoc
|
238
|
+
- MIT-LICENSE
|
218
239
|
- test/beetle/base_test.rb
|
219
240
|
- test/beetle/client_test.rb
|
220
241
|
- test/beetle/configuration_test.rb
|