beetle 0.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +18 -8
- data/beetle.gemspec +37 -121
- data/bin/beetle +9 -0
- data/examples/README.rdoc +0 -2
- data/examples/rpc.rb +3 -2
- data/ext/mkrf_conf.rb +19 -0
- data/lib/beetle/base.rb +1 -8
- data/lib/beetle/client.rb +16 -14
- data/lib/beetle/commands/configuration_client.rb +73 -0
- data/lib/beetle/commands/configuration_server.rb +85 -0
- data/lib/beetle/commands.rb +30 -0
- data/lib/beetle/configuration.rb +70 -7
- data/lib/beetle/deduplication_store.rb +50 -38
- data/lib/beetle/handler.rb +2 -5
- data/lib/beetle/logging.rb +7 -0
- data/lib/beetle/message.rb +11 -13
- data/lib/beetle/publisher.rb +2 -2
- data/lib/beetle/r_c.rb +2 -1
- data/lib/beetle/redis_configuration_client.rb +136 -0
- data/lib/beetle/redis_configuration_server.rb +301 -0
- data/lib/beetle/redis_ext.rb +79 -0
- data/lib/beetle/redis_master_file.rb +35 -0
- data/lib/beetle/redis_server_info.rb +65 -0
- data/lib/beetle/subscriber.rb +4 -1
- data/lib/beetle.rb +2 -2
- data/test/beetle/configuration_test.rb +14 -2
- data/test/beetle/deduplication_store_test.rb +61 -43
- data/test/beetle/message_test.rb +28 -4
- data/test/beetle/redis_configuration_client_test.rb +97 -0
- data/test/beetle/redis_configuration_server_test.rb +278 -0
- data/test/beetle/redis_ext_test.rb +71 -0
- data/test/beetle/redis_master_file_test.rb +39 -0
- data/test/test_helper.rb +13 -1
- metadata +59 -50
- data/.gitignore +0 -5
- data/MIT-LICENSE +0 -20
- data/Rakefile +0 -114
- data/TODO +0 -7
- data/doc/redundant_queues.graffle +0 -7744
- data/etc/redis-master.conf +0 -189
- data/etc/redis-slave.conf +0 -189
- data/examples/redis_failover.rb +0 -65
- data/script/start_rabbit +0 -29
- data/snafu.rb +0 -55
- data/test/beetle/bla.rb +0 -0
- data/test/beetle.yml +0 -81
- data/tmp/master/.gitignore +0 -2
- data/tmp/slave/.gitignore +0 -3
data/README.rdoc
CHANGED
@@ -18,9 +18,9 @@ More information can be found on the {project website}[http://xing.github.com/be
|
|
18
18
|
=== Configuration
|
19
19
|
# configure machines
|
20
20
|
|
21
|
-
Beetle.config do |
|
21
|
+
Beetle.config do |config|
|
22
22
|
config.servers = "broker1:5672, broker2:5672"
|
23
|
-
config.
|
23
|
+
config.redis_server = "redis1:6379"
|
24
24
|
end
|
25
25
|
|
26
26
|
# instantiate a beetle client
|
@@ -41,7 +41,19 @@ More information can be found on the {project website}[http://xing.github.com/be
|
|
41
41
|
=== Subscribing
|
42
42
|
b.listen
|
43
43
|
|
44
|
-
|
44
|
+
=== Examples
|
45
|
+
|
46
|
+
Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
|
47
|
+
|
48
|
+
The top level Rakefile comes with targets to start several RabbitMQ and redis instances
|
49
|
+
locally. Make sure the corresponding binaries are in your search path. Open four new shell
|
50
|
+
windows and execute the following commands:
|
51
|
+
|
52
|
+
rake rabbit:start1
|
53
|
+
rake rabbit:start2
|
54
|
+
rake redis:start1
|
55
|
+
rake redis:start2
|
56
|
+
|
45
57
|
|
46
58
|
== Prerequisites
|
47
59
|
|
@@ -68,8 +80,9 @@ For development, you'll need
|
|
68
80
|
== Authors
|
69
81
|
|
70
82
|
{Stefan Kaes}[http://github.com/skaes],
|
71
|
-
{Pascal Friederich}[http://github.com/paukul]
|
72
|
-
{Ali Jelveh}[http://github.com/dudemeister]
|
83
|
+
{Pascal Friederich}[http://github.com/paukul],
|
84
|
+
{Ali Jelveh}[http://github.com/dudemeister] and
|
85
|
+
{Sebastian Roebke}[http://github.com/boosty].
|
73
86
|
|
74
87
|
You cand find out more about our work on our {dev blog}[http://devblog.xing.com].
|
75
88
|
|
@@ -77,6 +90,3 @@ Copyright (c) 2010 {XING AG}[http://www.xing.com/]
|
|
77
90
|
|
78
91
|
Released under the MIT license. For full details see MIT-LICENSE included in this
|
79
92
|
distribution.
|
80
|
-
|
81
|
-
|
82
|
-
|
data/beetle.gemspec
CHANGED
@@ -1,127 +1,43 @@
|
|
1
|
-
# Generated by jeweler
|
2
|
-
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
-
# -*- encoding: utf-8 -*-
|
5
|
-
|
6
1
|
Gem::Specification.new do |s|
|
7
|
-
s.name
|
8
|
-
s.version = "0.
|
2
|
+
s.name = "beetle"
|
3
|
+
s.version = "0.2.0"
|
4
|
+
|
5
|
+
s.required_rubygems_version = ">= 1.3.1"
|
6
|
+
s.authors = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
|
7
|
+
s.date = Time.now.strftime('%Y-%m-%d')
|
8
|
+
s.default_executable = "beetle"
|
9
|
+
s.description = "A highly available, reliable messaging infrastructure"
|
10
|
+
s.summary = "High Availability AMQP Messaging with Redundant Queues"
|
11
|
+
s.email = "developers@xing.com"
|
12
|
+
s.executables = ["beetle"]
|
13
|
+
s.extra_rdoc_files = ["README.rdoc"]
|
14
|
+
s.files = Dir['{examples,ext,lib}/**/*.rb'] + %w(beetle.gemspec examples/README.rdoc)
|
15
|
+
s.extensions = 'ext/mkrf_conf.rb'
|
16
|
+
s.homepage = "http://xing.github.com/beetle/"
|
17
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
s.rubygems_version = "1.3.7"
|
20
|
+
s.test_files = Dir['test/**/*.rb']
|
21
|
+
|
22
|
+
s.post_install_message = <<-INFO
|
23
|
+
*********************************************************************************************
|
9
24
|
|
10
|
-
|
11
|
-
|
12
|
-
s.date = %q{2010-04-14}
|
13
|
-
s.description = %q{A highly available, reliable messaging infrastructure}
|
14
|
-
s.email = %q{developers@xing.com}
|
15
|
-
s.extra_rdoc_files = [
|
16
|
-
"README.rdoc",
|
17
|
-
"TODO"
|
18
|
-
]
|
19
|
-
s.files = [
|
20
|
-
".gitignore",
|
21
|
-
"MIT-LICENSE",
|
22
|
-
"README.rdoc",
|
23
|
-
"Rakefile",
|
24
|
-
"TODO",
|
25
|
-
"beetle.gemspec",
|
26
|
-
"doc/redundant_queues.graffle",
|
27
|
-
"etc/redis-master.conf",
|
28
|
-
"etc/redis-slave.conf",
|
29
|
-
"examples/README.rdoc",
|
30
|
-
"examples/attempts.rb",
|
31
|
-
"examples/handler_class.rb",
|
32
|
-
"examples/handling_exceptions.rb",
|
33
|
-
"examples/multiple_exchanges.rb",
|
34
|
-
"examples/multiple_queues.rb",
|
35
|
-
"examples/redis_failover.rb",
|
36
|
-
"examples/redundant.rb",
|
37
|
-
"examples/rpc.rb",
|
38
|
-
"examples/simple.rb",
|
39
|
-
"lib/beetle.rb",
|
40
|
-
"lib/beetle/base.rb",
|
41
|
-
"lib/beetle/client.rb",
|
42
|
-
"lib/beetle/configuration.rb",
|
43
|
-
"lib/beetle/deduplication_store.rb",
|
44
|
-
"lib/beetle/handler.rb",
|
45
|
-
"lib/beetle/message.rb",
|
46
|
-
"lib/beetle/publisher.rb",
|
47
|
-
"lib/beetle/r_c.rb",
|
48
|
-
"lib/beetle/subscriber.rb",
|
49
|
-
"script/start_rabbit",
|
50
|
-
"snafu.rb",
|
51
|
-
"test/beetle.yml",
|
52
|
-
"test/beetle/base_test.rb",
|
53
|
-
"test/beetle/bla.rb",
|
54
|
-
"test/beetle/client_test.rb",
|
55
|
-
"test/beetle/configuration_test.rb",
|
56
|
-
"test/beetle/deduplication_store_test.rb",
|
57
|
-
"test/beetle/handler_test.rb",
|
58
|
-
"test/beetle/message_test.rb",
|
59
|
-
"test/beetle/publisher_test.rb",
|
60
|
-
"test/beetle/r_c_test.rb",
|
61
|
-
"test/beetle/subscriber_test.rb",
|
62
|
-
"test/beetle_test.rb",
|
63
|
-
"test/test_helper.rb",
|
64
|
-
"tmp/master/.gitignore",
|
65
|
-
"tmp/slave/.gitignore"
|
66
|
-
]
|
67
|
-
s.homepage = %q{http://xing.github.com/beetle/}
|
68
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
69
|
-
s.require_paths = ["lib"]
|
70
|
-
s.rubygems_version = %q{1.3.5}
|
71
|
-
s.summary = %q{High Availability AMQP Messaging with Redundant Queues}
|
72
|
-
s.test_files = [
|
73
|
-
"test/beetle/base_test.rb",
|
74
|
-
"test/beetle/bla.rb",
|
75
|
-
"test/beetle/client_test.rb",
|
76
|
-
"test/beetle/configuration_test.rb",
|
77
|
-
"test/beetle/deduplication_store_test.rb",
|
78
|
-
"test/beetle/handler_test.rb",
|
79
|
-
"test/beetle/message_test.rb",
|
80
|
-
"test/beetle/publisher_test.rb",
|
81
|
-
"test/beetle/r_c_test.rb",
|
82
|
-
"test/beetle/subscriber_test.rb",
|
83
|
-
"test/beetle_test.rb",
|
84
|
-
"test/test_helper.rb",
|
85
|
-
"examples/attempts.rb",
|
86
|
-
"examples/handler_class.rb",
|
87
|
-
"examples/handling_exceptions.rb",
|
88
|
-
"examples/multiple_exchanges.rb",
|
89
|
-
"examples/multiple_queues.rb",
|
90
|
-
"examples/redis_failover.rb",
|
91
|
-
"examples/redundant.rb",
|
92
|
-
"examples/rpc.rb",
|
93
|
-
"examples/simple.rb"
|
94
|
-
]
|
25
|
+
If you're running a ruby version < 1.9 we silently installed the SystemTimer gem for you.
|
26
|
+
See: http://ph7spot.com/musings/system-timer
|
95
27
|
|
96
|
-
|
97
|
-
|
98
|
-
s.specification_version = 3
|
28
|
+
*********************************************************************************************
|
29
|
+
INFO
|
99
30
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
s.add_dependency(%q<redis>, [">= 0.1.2"])
|
112
|
-
s.add_dependency(%q<amqp>, [">= 0.6.7"])
|
113
|
-
s.add_dependency(%q<activesupport>, [">= 2.3.4"])
|
114
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
115
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
116
|
-
end
|
117
|
-
else
|
118
|
-
s.add_dependency(%q<uuid4r>, [">= 0.1.1"])
|
119
|
-
s.add_dependency(%q<bunny>, [">= 0.6.0"])
|
120
|
-
s.add_dependency(%q<redis>, [">= 0.1.2"])
|
121
|
-
s.add_dependency(%q<amqp>, [">= 0.6.7"])
|
122
|
-
s.add_dependency(%q<activesupport>, [">= 2.3.4"])
|
123
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
124
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
125
|
-
end
|
31
|
+
s.specification_version = 3
|
32
|
+
s.add_runtime_dependency("uuid4r", [">= 0.1.1"])
|
33
|
+
s.add_runtime_dependency("bunny", [">= 0.6.0"])
|
34
|
+
s.add_runtime_dependency("redis", [">= 2.0.3"])
|
35
|
+
s.add_runtime_dependency("amqp", [">= 0.6.7"])
|
36
|
+
s.add_runtime_dependency("activesupport", [">= 2.3.4"])
|
37
|
+
s.add_runtime_dependency("daemons", [">= 1.0.10"])
|
38
|
+
s.add_development_dependency("mocha", [">= 0"])
|
39
|
+
s.add_development_dependency("rcov", [">= 0"])
|
40
|
+
s.add_development_dependency("cucumber", [">= 0.7.2"])
|
41
|
+
s.add_development_dependency("daemon_controller", [">= 0"])
|
126
42
|
end
|
127
43
|
|
data/bin/beetle
ADDED
data/examples/README.rdoc
CHANGED
data/examples/rpc.rb
CHANGED
@@ -4,9 +4,10 @@ require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
|
|
4
4
|
|
5
5
|
# suppress debug messages
|
6
6
|
Beetle.config.logger.level = Logger::DEBUG
|
7
|
-
|
7
|
+
Beetle.config.servers = "localhost:5672, localhost:5673"
|
8
8
|
# instantiate a client
|
9
|
-
|
9
|
+
|
10
|
+
client = Beetle::Client.new
|
10
11
|
|
11
12
|
# register a durable queue named 'test'
|
12
13
|
# this implicitly registers a durable topic exchange called 'test'
|
data/ext/mkrf_conf.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rubygems/command.rb'
|
3
|
+
require 'rubygems/dependency_installer.rb'
|
4
|
+
begin
|
5
|
+
Gem::Command.build_args = ARGV
|
6
|
+
rescue NoMethodError
|
7
|
+
end
|
8
|
+
inst = Gem::DependencyInstaller.new
|
9
|
+
begin
|
10
|
+
if RUBY_VERSION < "1.9"
|
11
|
+
inst.install "SystemTimer", ">= 1.2"
|
12
|
+
end
|
13
|
+
rescue
|
14
|
+
exit(1)
|
15
|
+
end
|
16
|
+
|
17
|
+
f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w") # create dummy rakefile to indicate success
|
18
|
+
f.write("task :default\n")
|
19
|
+
f.close
|
data/lib/beetle/base.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module Beetle
|
2
2
|
# Abstract base class shared by Publisher and Subscriber
|
3
3
|
class Base
|
4
|
+
include Logging
|
4
5
|
|
5
6
|
attr_accessor :options, :servers, :server #:nodoc:
|
6
7
|
|
@@ -15,14 +16,6 @@ module Beetle
|
|
15
16
|
|
16
17
|
private
|
17
18
|
|
18
|
-
def logger
|
19
|
-
self.class.logger
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.logger
|
23
|
-
Beetle.config.logger
|
24
|
-
end
|
25
|
-
|
26
19
|
def error(text)
|
27
20
|
logger.error text
|
28
21
|
raise Error.new(text)
|
data/lib/beetle/client.rb
CHANGED
@@ -19,6 +19,8 @@ module Beetle
|
|
19
19
|
# order, so that no message is lost if message producers are accidentally started before
|
20
20
|
# the corresponding consumers.
|
21
21
|
class Client
|
22
|
+
include Logging
|
23
|
+
|
22
24
|
# the AMQP servers available for publishing
|
23
25
|
attr_reader :servers
|
24
26
|
|
@@ -37,14 +39,18 @@ module Beetle
|
|
37
39
|
# the deduplication store to use for this client
|
38
40
|
attr_reader :deduplication_store
|
39
41
|
|
42
|
+
# accessor for the beetle configuration
|
43
|
+
attr_reader :config
|
44
|
+
|
40
45
|
# create a fresh Client instance from a given configuration object
|
41
46
|
def initialize(config = Beetle.config)
|
47
|
+
@config = config
|
42
48
|
@servers = config.servers.split(/ *, */)
|
43
49
|
@exchanges = {}
|
44
50
|
@queues = {}
|
45
51
|
@messages = {}
|
46
52
|
@bindings = {}
|
47
|
-
@deduplication_store = DeduplicationStore.new(config
|
53
|
+
@deduplication_store = DeduplicationStore.new(config)
|
48
54
|
end
|
49
55
|
|
50
56
|
# register an exchange with the given _name_ and a set of _options_:
|
@@ -69,8 +75,8 @@ module Beetle
|
|
69
75
|
def register_queue(name, options={})
|
70
76
|
name = name.to_s
|
71
77
|
raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
|
72
|
-
opts = {:exchange => name, :key => name}.merge!(options.symbolize_keys)
|
73
|
-
opts.merge! :durable => true, :passive => false, :exclusive => false
|
78
|
+
opts = {:exchange => name, :key => name, :auto_delete => false, :amqp_name => name}.merge!(options.symbolize_keys)
|
79
|
+
opts.merge! :durable => true, :passive => false, :exclusive => false
|
74
80
|
exchange = opts.delete(:exchange).to_s
|
75
81
|
key = opts.delete(:key)
|
76
82
|
queues[name] = opts
|
@@ -139,10 +145,10 @@ module Beetle
|
|
139
145
|
|
140
146
|
# this is a convenience method to configure exchanges, queues, messages and handlers
|
141
147
|
# with a common set of options. allows one to call all register methods without the
|
142
|
-
# register_ prefix.
|
148
|
+
# register_ prefix. returns self.
|
143
149
|
#
|
144
150
|
# Example:
|
145
|
-
# client.configure :exchange => :foobar do |config|
|
151
|
+
# client = Beetle.client.new.configure :exchange => :foobar do |config|
|
146
152
|
# config.queue :q1, :key => "foo"
|
147
153
|
# config.queue :q2, :key => "bar"
|
148
154
|
# config.message :foo
|
@@ -152,6 +158,7 @@ module Beetle
|
|
152
158
|
# end
|
153
159
|
def configure(options={}) #:yields: config
|
154
160
|
yield Configurator.new(self, options)
|
161
|
+
self
|
155
162
|
end
|
156
163
|
|
157
164
|
# publishes a message. the given options hash is merged with options given on message registration.
|
@@ -195,10 +202,10 @@ module Beetle
|
|
195
202
|
publisher.stop
|
196
203
|
end
|
197
204
|
|
198
|
-
# traces
|
199
|
-
def trace(&block)
|
205
|
+
# traces messages without consuming them. useful for debugging message flow.
|
206
|
+
def trace(messages=self.messages.keys, &block)
|
200
207
|
queues.each do |name, opts|
|
201
|
-
opts.merge! :durable => false, :auto_delete => true, :amqp_name => queue_name_for_tracing(
|
208
|
+
opts.merge! :durable => false, :auto_delete => true, :amqp_name => queue_name_for_tracing(opts[:amqp_name])
|
202
209
|
end
|
203
210
|
register_handler(queues.keys) do |msg|
|
204
211
|
puts "-----===== new message =====-----"
|
@@ -207,7 +214,7 @@ module Beetle
|
|
207
214
|
puts "MSGID: #{msg.msg_id}"
|
208
215
|
puts "DATA: #{msg.data}"
|
209
216
|
end
|
210
|
-
|
217
|
+
listen(messages, &block)
|
211
218
|
end
|
212
219
|
|
213
220
|
# evaluate the ruby files matching the given +glob+ pattern in the context of the client instance.
|
@@ -218,11 +225,6 @@ module Beetle
|
|
218
225
|
end
|
219
226
|
end
|
220
227
|
|
221
|
-
# returns the configured Logger instance
|
222
|
-
def logger
|
223
|
-
@logger ||= Beetle.config.logger
|
224
|
-
end
|
225
|
-
|
226
228
|
private
|
227
229
|
|
228
230
|
class Configurator #:nodoc:all
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'daemons'
|
3
|
+
require 'beetle'
|
4
|
+
|
5
|
+
module Beetle
|
6
|
+
module Commands
|
7
|
+
# Command to start a RedisConfigurationClient daemon.
|
8
|
+
#
|
9
|
+
# Usage: beetle configuration_client [options] -- [client options]
|
10
|
+
#
|
11
|
+
# client options:
|
12
|
+
# --redis-master-file FILE Write redis master server string to FILE
|
13
|
+
# --id, --client-id ID Set unique client id (default is minastirith.local)
|
14
|
+
# --amqp-servers LIST AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
|
15
|
+
# --config-file PATH Path to an external yaml config file
|
16
|
+
# --pid-dir DIR Write pid and log to DIR
|
17
|
+
# -v, --verbose Set log level to DEBUG
|
18
|
+
# -h, --help Show this message
|
19
|
+
#
|
20
|
+
class ConfigurationClient
|
21
|
+
# parses command line options and starts Beetle::RedisConfigurationClient as a daemon
|
22
|
+
def self.execute
|
23
|
+
command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
|
24
|
+
|
25
|
+
opts = OptionParser.new
|
26
|
+
opts.banner = "Usage: beetle configuration_client #{command} [options] -- [client options]"
|
27
|
+
opts.separator ""
|
28
|
+
opts.separator "client options:"
|
29
|
+
|
30
|
+
opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
|
31
|
+
Beetle.config.redis_server = val
|
32
|
+
end
|
33
|
+
|
34
|
+
client_id = nil
|
35
|
+
opts.on("--id ID", "--client-id ID", String, "Set unique client id (default is #{RedisConfigurationClient.new.id})") do |val|
|
36
|
+
client_id = val
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
|
40
|
+
Beetle.config.servers = val
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
|
44
|
+
Beetle.config.config_file = val
|
45
|
+
end
|
46
|
+
|
47
|
+
dir_mode = nil
|
48
|
+
dir = nil
|
49
|
+
opts.on("--pid-dir DIR", String, "Write pid and log to DIR") do |val|
|
50
|
+
dir_mode = :normal
|
51
|
+
dir = val
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on("-v", "--verbose", "Set log level to DEBUG") do |val|
|
55
|
+
Beetle.config.logger.level = Logger::DEBUG
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
59
|
+
puts opts
|
60
|
+
exit
|
61
|
+
end
|
62
|
+
|
63
|
+
opts.parse!(app_options)
|
64
|
+
|
65
|
+
Daemons.run_proc("redis_configuration_client", :multiple => true, :log_output => true, :dir_mode => dir_mode, :dir => dir) do
|
66
|
+
client = Beetle::RedisConfigurationClient.new
|
67
|
+
client.id = client_id if client_id
|
68
|
+
client.start
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'daemons'
|
3
|
+
require 'beetle'
|
4
|
+
|
5
|
+
module Beetle
|
6
|
+
module Commands
|
7
|
+
# Command to start a RedisConfigurationServer daemon.
|
8
|
+
#
|
9
|
+
# Usage: beetle configuration_server [options] -- [server options]
|
10
|
+
#
|
11
|
+
# server options:
|
12
|
+
# --redis-servers LIST Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)
|
13
|
+
# --client-ids LIST Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)
|
14
|
+
# --redis-master-file FILE Write redis master server string to FILE
|
15
|
+
# --redis-retry-interval SEC Number of seconds to wait between master checks
|
16
|
+
# --amqp-servers LIST AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
|
17
|
+
# --config-file PATH Path to an external yaml config file
|
18
|
+
# --pid-dir DIR Write pid and log to DIR
|
19
|
+
# -v, --verbose
|
20
|
+
# -h, --help Show this message
|
21
|
+
#
|
22
|
+
class ConfigurationServer
|
23
|
+
# parses command line options and starts Beetle::RedisConfigurationServer as a daemon
|
24
|
+
def self.execute
|
25
|
+
command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
|
26
|
+
|
27
|
+
opts = OptionParser.new
|
28
|
+
opts.banner = "Usage: beetle configuration_server #{command} [options] -- [server options]"
|
29
|
+
opts.separator ""
|
30
|
+
opts.separator "server options:"
|
31
|
+
|
32
|
+
opts.on("--redis-servers LIST", Array, "Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
|
33
|
+
Beetle.config.redis_servers = val.join(",")
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on("--client-ids LIST", "Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)") do |val|
|
37
|
+
Beetle.config.redis_configuration_client_ids = val
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
|
41
|
+
Beetle.config.redis_server = val
|
42
|
+
end
|
43
|
+
|
44
|
+
opts.on("--redis-retry-interval SEC", Integer, "Number of seconds to wait between master checks") do |val|
|
45
|
+
Beetle.config.redis_configuration_master_retry_interval = val
|
46
|
+
end
|
47
|
+
|
48
|
+
opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
|
49
|
+
Beetle.config.servers = val
|
50
|
+
end
|
51
|
+
|
52
|
+
opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
|
53
|
+
Beetle.config.config_file = val
|
54
|
+
end
|
55
|
+
|
56
|
+
dir_mode = nil
|
57
|
+
dir = nil
|
58
|
+
opts.on("--pid-dir DIR", String, "Write pid and log to DIR") do |val|
|
59
|
+
dir_mode = :normal
|
60
|
+
dir = val
|
61
|
+
end
|
62
|
+
|
63
|
+
opts.on("-v", "--verbose") do |val|
|
64
|
+
Beetle.config.logger.level = Logger::DEBUG
|
65
|
+
end
|
66
|
+
|
67
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
68
|
+
puts opts
|
69
|
+
exit
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.parse!(app_options)
|
73
|
+
|
74
|
+
if command =~ /start|run/ && Beetle.config.redis_servers.blank?
|
75
|
+
puts opts
|
76
|
+
exit
|
77
|
+
end
|
78
|
+
|
79
|
+
Daemons.run_proc("redis_configuration_server", :log_output => true, :dir_mode => dir_mode, :dir => dir) do
|
80
|
+
Beetle::RedisConfigurationServer.new.start
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'active_support'
|
3
|
+
|
4
|
+
module Beetle
|
5
|
+
module Commands
|
6
|
+
# invokes given command by instantiating an appropriate command class
|
7
|
+
def self.execute(command)
|
8
|
+
if commands.include? command
|
9
|
+
require File.expand_path("../commands/#{command}", __FILE__)
|
10
|
+
"Beetle::Commands::#{command.classify}".constantize.execute
|
11
|
+
else
|
12
|
+
# me no likez no frikin heredocs
|
13
|
+
puts "\nCommand #{command} not known\n" if command
|
14
|
+
puts "Available commands are:"
|
15
|
+
puts
|
16
|
+
commands.each {|c| puts "\t #{c}"}
|
17
|
+
puts
|
18
|
+
exit 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
def self.commands
|
24
|
+
commands_dir = File.expand_path('../commands', __FILE__)
|
25
|
+
Dir[commands_dir + '/*.rb'].map {|f| File.basename(f)[0..-4]}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Beetle::Commands.execute(ARGV.shift)
|
data/lib/beetle/configuration.rb
CHANGED
@@ -1,31 +1,94 @@
|
|
1
|
+
require 'erb'
|
2
|
+
|
1
3
|
module Beetle
|
2
4
|
class Configuration
|
5
|
+
# system name (used for redis cluster partitioning) (defaults to <tt>system</tt>)
|
6
|
+
attr_accessor :system_name
|
3
7
|
# default logger (defaults to <tt>Logger.new(STDOUT)</tt>)
|
4
8
|
attr_accessor :logger
|
5
9
|
# number of seconds after which keys are removed form the message deduplication store (defaults to <tt>3.days</tt>)
|
6
10
|
attr_accessor :gc_threshold
|
7
|
-
# the
|
8
|
-
|
11
|
+
# the redis server to use for deduplication
|
12
|
+
# either a string like <tt>"localhost:6379"</tt> (default) or a file that contains the string.
|
13
|
+
# use a file if you are using a beetle configuration_client process to update it for automatic redis failover.
|
14
|
+
attr_accessor :redis_server
|
15
|
+
# comma separated list of redis servers available for master/slave switching
|
16
|
+
# e.g. "192.168.1.2:6379,192.168.1.3:6379"
|
17
|
+
attr_accessor :redis_servers
|
9
18
|
# redis database number to use for the message deduplication store (defaults to <tt>4</tt>)
|
10
19
|
attr_accessor :redis_db
|
20
|
+
|
21
|
+
# how long we should repeatedly retry a redis operation before giving up, with a one
|
22
|
+
# second sleep between retries (defaults to <tt>180.seconds</tt>). this value needs to be
|
23
|
+
# somewehere between the maximum time it takes to auto-switch redis and the smallest
|
24
|
+
# handler timeout.
|
25
|
+
attr_accessor :redis_failover_timeout
|
26
|
+
|
27
|
+
## redis configuration server options
|
28
|
+
# how often should the redis configuration server try to reach the redis master before nominating a new one (defaults to <tt>3</tt>)
|
29
|
+
attr_accessor :redis_configuration_master_retries
|
30
|
+
# number of seconds to wait between retries (defaults to <tt>10</tt>)
|
31
|
+
attr_accessor :redis_configuration_master_retry_interval
|
32
|
+
# number of seconds the redis configuration server waits for answers from clients (defaults to <tt>5</tt>)
|
33
|
+
attr_accessor :redis_configuration_client_timeout
|
34
|
+
# the redis configuration client ids living on the worker machines taking part in the redis failover, separated by comma (defaults to <tt>""</tt>)
|
35
|
+
attr_accessor :redis_configuration_client_ids
|
36
|
+
|
11
37
|
# list of amqp servers to use (defaults to <tt>"localhost:5672"</tt>)
|
12
38
|
attr_accessor :servers
|
13
|
-
# the virtual host to use on the AMQP servers
|
39
|
+
# the virtual host to use on the AMQP servers (defaults to <tt>"/"</tt>)
|
14
40
|
attr_accessor :vhost
|
15
|
-
# the AMQP user to use when connecting to the AMQP servers
|
41
|
+
# the AMQP user to use when connecting to the AMQP servers (defaults to <tt>"guest"</tt>)
|
16
42
|
attr_accessor :user
|
17
|
-
# the password to use when connectiong to the AMQP servers
|
43
|
+
# the password to use when connectiong to the AMQP servers (defaults to <tt>"guest"</tt>)
|
18
44
|
attr_accessor :password
|
19
45
|
|
46
|
+
# external config file (defaults to <tt>no file</tt>)
|
47
|
+
attr_reader :config_file
|
48
|
+
|
20
49
|
def initialize #:nodoc:
|
21
|
-
self.
|
50
|
+
self.system_name = "system"
|
51
|
+
|
52
|
+
self.logger = begin
|
53
|
+
logger = Logger.new(STDOUT)
|
54
|
+
logger.formatter = Logger::Formatter.new
|
55
|
+
logger.level = Logger::INFO
|
56
|
+
logger.datetime_format = "%Y-%m-%d %H:%M:%S"
|
57
|
+
logger
|
58
|
+
end
|
59
|
+
|
22
60
|
self.gc_threshold = 3.days
|
23
|
-
self.
|
61
|
+
self.redis_server = "localhost:6379"
|
62
|
+
self.redis_servers = ""
|
24
63
|
self.redis_db = 4
|
64
|
+
self.redis_failover_timeout = 180.seconds
|
65
|
+
|
66
|
+
self.redis_configuration_master_retries = 3
|
67
|
+
self.redis_configuration_master_retry_interval = 10.seconds
|
68
|
+
self.redis_configuration_client_timeout = 5.seconds
|
69
|
+
self.redis_configuration_client_ids = ""
|
70
|
+
|
25
71
|
self.servers = "localhost:5672"
|
26
72
|
self.vhost = "/"
|
27
73
|
self.user = "guest"
|
28
74
|
self.password = "guest"
|
29
75
|
end
|
76
|
+
|
77
|
+
# setting the external config file will load it on assignment
|
78
|
+
def config_file=(file_name) #:nodoc:
|
79
|
+
@config_file = file_name
|
80
|
+
load_config
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
def load_config
|
85
|
+
hash = YAML::load(ERB.new(IO.read(config_file)).result)
|
86
|
+
hash.each do |key, value|
|
87
|
+
send("#{key}=", value)
|
88
|
+
end
|
89
|
+
rescue Exception
|
90
|
+
logger.error "Error loading beetle config file '#{config_file}': #{$!}"
|
91
|
+
raise
|
92
|
+
end
|
30
93
|
end
|
31
94
|
end
|