beetle 1.0.3 → 2.0.0rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +15 -7
- data/REDIS_AUTO_FAILOVER.rdoc +8 -16
- data/RELEASE_NOTES.rdoc +9 -0
- data/beetle.gemspec +22 -10
- data/features/support/system_notification_logger +29 -12
- data/features/support/test_daemons/redis.rb +1 -1
- data/features/support/test_daemons/redis_configuration_client.rb +3 -3
- data/features/support/test_daemons/redis_configuration_server.rb +2 -2
- data/lib/beetle/deduplication_store.rb +0 -66
- data/lib/beetle/message.rb +1 -1
- data/lib/beetle/publisher.rb +2 -1
- data/lib/beetle/redis_master_file.rb +0 -5
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +0 -48
- data/test/beetle/message_test.rb +1 -31
- data/test/beetle/publisher_test.rb +2 -1
- metadata +165 -26
- data/bin/beetle +0 -9
- data/lib/beetle/commands.rb +0 -35
- data/lib/beetle/commands/configuration_client.rb +0 -98
- data/lib/beetle/commands/configuration_server.rb +0 -98
- data/lib/beetle/commands/garbage_collect_deduplication_store.rb +0 -52
- data/lib/beetle/redis_configuration_client.rb +0 -157
- data/lib/beetle/redis_configuration_http_server.rb +0 -152
- data/lib/beetle/redis_configuration_server.rb +0 -438
- data/lib/beetle/redis_server_info.rb +0 -66
- data/script/docker-run-beetle-tests- +0 -5
- data/test/beetle/redis_configuration_client_test.rb +0 -118
- data/test/beetle/redis_configuration_server_test.rb +0 -381
@@ -1,98 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
require 'daemons'
|
3
|
-
require 'beetle'
|
4
|
-
|
5
|
-
module Beetle
|
6
|
-
module Commands
|
7
|
-
# Command to start a RedisConfigurationServer daemon.
|
8
|
-
#
|
9
|
-
# Usage: beetle configuration_server [options] -- [server options]
|
10
|
-
#
|
11
|
-
# server options:
|
12
|
-
# --redis-servers LIST Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)
|
13
|
-
# --client-ids LIST Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)
|
14
|
-
# --redis-master-file FILE Write redis master server string to FILE
|
15
|
-
# --redis-retry-interval SEC Number of seconds to wait between master checks
|
16
|
-
# --amqp-servers LIST AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
|
17
|
-
# --config-file PATH Path to an external yaml config file
|
18
|
-
# --pid-dir DIR Write pid and log to DIR
|
19
|
-
# -v, --verbose
|
20
|
-
# -h, --help Show this message
|
21
|
-
#
|
22
|
-
class ConfigurationServer
|
23
|
-
# parses command line options and starts Beetle::RedisConfigurationServer as a daemon
|
24
|
-
def self.execute
|
25
|
-
command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
|
26
|
-
|
27
|
-
opts = OptionParser.new
|
28
|
-
opts.banner = "Usage: beetle configuration_server #{command} [options] -- [server options]"
|
29
|
-
opts.separator ""
|
30
|
-
opts.separator "server options:"
|
31
|
-
|
32
|
-
opts.on("--redis-servers LIST", Array, "Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
|
33
|
-
Beetle.config.redis_servers = val.join(",")
|
34
|
-
end
|
35
|
-
|
36
|
-
opts.on("--client-ids LIST", "Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)") do |val|
|
37
|
-
Beetle.config.redis_configuration_client_ids = val
|
38
|
-
end
|
39
|
-
|
40
|
-
opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
|
41
|
-
Beetle.config.redis_server = val
|
42
|
-
end
|
43
|
-
|
44
|
-
opts.on("--redis-retry-interval SEC", Integer, "Number of seconds to wait between master checks") do |val|
|
45
|
-
Beetle.config.redis_configuration_master_retry_interval = val
|
46
|
-
end
|
47
|
-
|
48
|
-
opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
|
49
|
-
Beetle.config.servers = val
|
50
|
-
end
|
51
|
-
|
52
|
-
opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
|
53
|
-
Beetle.config.config_file = val
|
54
|
-
end
|
55
|
-
|
56
|
-
dir_mode = nil
|
57
|
-
dir = nil
|
58
|
-
opts.on("--pid-dir DIR", String, "Write pid and output to DIR") do |val|
|
59
|
-
dir_mode = :normal
|
60
|
-
dir = val
|
61
|
-
end
|
62
|
-
|
63
|
-
opts.on("-v", "--verbose") do |val|
|
64
|
-
Beetle.config.logger.level = Logger::DEBUG
|
65
|
-
end
|
66
|
-
|
67
|
-
opts.on_tail("-h", "--help", "Show this message") do
|
68
|
-
puts opts
|
69
|
-
exit
|
70
|
-
end
|
71
|
-
|
72
|
-
opts.parse!(app_options)
|
73
|
-
|
74
|
-
if command =~ /start|run/ && Beetle.config.redis_servers.blank?
|
75
|
-
puts opts
|
76
|
-
exit
|
77
|
-
end
|
78
|
-
|
79
|
-
daemon_options = {
|
80
|
-
:log_output => true,
|
81
|
-
:dir_mode => dir_mode,
|
82
|
-
:dir => dir,
|
83
|
-
:force => true
|
84
|
-
}
|
85
|
-
|
86
|
-
Daemons.run_proc("redis_configuration_server", daemon_options) do
|
87
|
-
config_server = Beetle::RedisConfigurationServer.new
|
88
|
-
Beetle::RedisConfigurationHttpServer.config_server = config_server
|
89
|
-
http_server_port = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
|
90
|
-
EM.run do
|
91
|
-
config_server.start
|
92
|
-
EM.start_server '0.0.0.0', http_server_port, Beetle::RedisConfigurationHttpServer
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
require 'beetle'
|
3
|
-
|
4
|
-
module Beetle
|
5
|
-
module Commands
|
6
|
-
# Command to garbage collect the deduplication store
|
7
|
-
#
|
8
|
-
# Usage: beetle garbage_collect_deduplication_store [options]
|
9
|
-
#
|
10
|
-
# options:
|
11
|
-
# --redis-servers LIST Required (e.g. 192.168.0.1:6379,192.168.0.2:6379)
|
12
|
-
# --config-file PATH Path to an external yaml config file
|
13
|
-
# -v, --verbose
|
14
|
-
# -h, --help Show this message
|
15
|
-
#
|
16
|
-
class GarbageCollectDeduplicationStore
|
17
|
-
# parses command line options and starts Beetle::RedisConfigurationServer as a daemon
|
18
|
-
def self.execute
|
19
|
-
opts = OptionParser.new
|
20
|
-
opts.banner = "Usage: beetle garbage_collect_deduplication_store [options]"
|
21
|
-
opts.separator ""
|
22
|
-
|
23
|
-
opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
|
24
|
-
Beetle.config.config_file = val
|
25
|
-
Beetle.config.log_file = STDOUT
|
26
|
-
end
|
27
|
-
|
28
|
-
opts.on("--redis-servers LIST", Array, "Comma separted list of redis server:port specs used for GC") do |val|
|
29
|
-
Beetle.config.redis_servers = val.join(",")
|
30
|
-
end
|
31
|
-
|
32
|
-
opts.on("--redis-db N", Integer, "Redis database used for GC") do |val|
|
33
|
-
Beetle.config.redis_db = val.to_i
|
34
|
-
end
|
35
|
-
|
36
|
-
opts.on("-v", "--verbose") do |val|
|
37
|
-
Beetle.config.log_file = STDOUT
|
38
|
-
Beetle.config.logger.level = Logger::DEBUG
|
39
|
-
end
|
40
|
-
|
41
|
-
opts.on_tail("-h", "--help", "Show this message") do
|
42
|
-
puts opts
|
43
|
-
exit
|
44
|
-
end
|
45
|
-
|
46
|
-
opts.parse!(ARGV)
|
47
|
-
|
48
|
-
DeduplicationStore.new.garbage_collect_keys_using_master_and_slave
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
@@ -1,157 +0,0 @@
|
|
1
|
-
module Beetle
|
2
|
-
# A RedisConfigurationClient is the subordinate part of beetle's
|
3
|
-
# redis failover solution
|
4
|
-
#
|
5
|
-
# An instance of RedisConfigurationClient lives on every server that
|
6
|
-
# hosts message consumers (worker server).
|
7
|
-
#
|
8
|
-
# It is responsible for determining an initial redis master and reacting to redis master
|
9
|
-
# switches initiated by the RedisConfigurationServer.
|
10
|
-
#
|
11
|
-
# It will write the current redis master host:port string to a file specified via a
|
12
|
-
# Configuration, which is then read by DeduplicationStore on redis access.
|
13
|
-
#
|
14
|
-
# Usually started via <tt>beetle configuration_client</tt> command.
|
15
|
-
class RedisConfigurationClient
|
16
|
-
include Logging
|
17
|
-
include RedisMasterFile
|
18
|
-
|
19
|
-
# Set a custom unique id for this instance. Must match an entry in
|
20
|
-
# Configuration#redis_configuration_client_ids.
|
21
|
-
attr_writer :id
|
22
|
-
|
23
|
-
# The current redis master
|
24
|
-
attr_reader :current_master
|
25
|
-
|
26
|
-
# Unique id for this instance (defaults to the fully qualified hostname)
|
27
|
-
def id
|
28
|
-
@id ||= Beetle.hostname
|
29
|
-
end
|
30
|
-
|
31
|
-
def initialize #:nodoc:
|
32
|
-
@current_token = nil
|
33
|
-
MessageDispatcher.configuration_client = self
|
34
|
-
end
|
35
|
-
|
36
|
-
# determinines the initial redis master (if possible), then enters a messaging event
|
37
|
-
# loop, reacting to failover related messages sent by RedisConfigurationServer.
|
38
|
-
def start
|
39
|
-
verify_redis_master_file_string
|
40
|
-
client_started!
|
41
|
-
logger.info "RedisConfigurationClient starting (client id: #{id})"
|
42
|
-
determine_initial_master
|
43
|
-
clear_redis_master_file unless current_master.try(:master?)
|
44
|
-
logger.info "Listening"
|
45
|
-
beetle.listen do
|
46
|
-
EventMachine.add_periodic_timer(Beetle.config.redis_failover_client_heartbeat_interval) do
|
47
|
-
heartbeat!
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
# called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
|
53
|
-
def ping(payload)
|
54
|
-
token = payload["token"]
|
55
|
-
logger.info "Received ping message with token '#{token}'"
|
56
|
-
pong! if redeem_token(token)
|
57
|
-
end
|
58
|
-
|
59
|
-
# called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
|
60
|
-
def invalidate(payload)
|
61
|
-
token = payload["token"]
|
62
|
-
logger.info "Received invalidate message with token '#{token}'"
|
63
|
-
invalidate! if redeem_token(token) && !current_master.try(:master?)
|
64
|
-
end
|
65
|
-
|
66
|
-
# called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
|
67
|
-
def reconfigure(payload)
|
68
|
-
server = payload["server"]
|
69
|
-
token = payload["token"]
|
70
|
-
logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
|
71
|
-
return unless redeem_token(token)
|
72
|
-
unless server == read_redis_master_file
|
73
|
-
new_master!(server)
|
74
|
-
write_redis_master_file(server)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Beetle::Client instance for communication with the RedisConfigurationServer
|
79
|
-
def beetle
|
80
|
-
@beetle ||= build_beetle
|
81
|
-
end
|
82
|
-
|
83
|
-
def config #:nodoc:
|
84
|
-
beetle.config
|
85
|
-
end
|
86
|
-
|
87
|
-
private
|
88
|
-
|
89
|
-
def determine_initial_master
|
90
|
-
if master_file_exists? && server = read_redis_master_file
|
91
|
-
new_master!(server)
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
def new_master!(server)
|
96
|
-
@current_master = Redis.from_server_string(server, :timeout => 3)
|
97
|
-
end
|
98
|
-
|
99
|
-
def build_beetle
|
100
|
-
system = Beetle.config.system_name
|
101
|
-
Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
|
102
|
-
# messages sent
|
103
|
-
config.message :pong
|
104
|
-
config.message :client_started
|
105
|
-
config.message :client_invalidated
|
106
|
-
config.message :heartbeat
|
107
|
-
# messages received
|
108
|
-
config.message :ping
|
109
|
-
config.message :invalidate
|
110
|
-
config.message :reconfigure
|
111
|
-
# queue setup
|
112
|
-
config.queue :client, :key => 'ping', :amqp_name => "#{system}_configuration_client_#{id}"
|
113
|
-
config.binding :client, :key => 'invalidate'
|
114
|
-
config.binding :client, :key => 'reconfigure'
|
115
|
-
config.handler :client, MessageDispatcher
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
def redeem_token(token)
|
120
|
-
@current_token = token if @current_token.nil? || token > @current_token
|
121
|
-
token_valid = token >= @current_token
|
122
|
-
logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
|
123
|
-
token_valid
|
124
|
-
end
|
125
|
-
|
126
|
-
def pong!
|
127
|
-
logger.info "Sending pong message with id '#{id}' and token '#{@current_token}'"
|
128
|
-
beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
|
129
|
-
end
|
130
|
-
|
131
|
-
def client_started!
|
132
|
-
logger.info "Sending client_started message with id '#{id}'"
|
133
|
-
beetle.publish(:client_started, {"id" => id}.to_json)
|
134
|
-
end
|
135
|
-
|
136
|
-
def heartbeat!
|
137
|
-
logger.info "Sending heartbeat message with id '#{id}'"
|
138
|
-
beetle.publish(:heartbeat, {"id" => id}.to_json)
|
139
|
-
end
|
140
|
-
|
141
|
-
def invalidate!
|
142
|
-
@current_master = nil
|
143
|
-
clear_redis_master_file
|
144
|
-
logger.info "Sending client_invalidated message with id '#{id}' and token '#{@current_token}'"
|
145
|
-
beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
|
146
|
-
end
|
147
|
-
|
148
|
-
|
149
|
-
# Dispatches messages from the queue to methods in RedisConfigurationClient
|
150
|
-
class MessageDispatcher < Beetle::Handler #:nodoc:
|
151
|
-
cattr_accessor :configuration_client
|
152
|
-
def process
|
153
|
-
@@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
|
154
|
-
end
|
155
|
-
end
|
156
|
-
end
|
157
|
-
end
|
@@ -1,152 +0,0 @@
|
|
1
|
-
require 'evma_httpserver'
|
2
|
-
|
3
|
-
module Beetle
|
4
|
-
class RedisConfigurationHttpServer < EM::Connection
|
5
|
-
include EM::HttpServer
|
6
|
-
|
7
|
-
def post_init
|
8
|
-
super
|
9
|
-
no_environment_strings
|
10
|
-
end
|
11
|
-
|
12
|
-
cattr_accessor :config_server
|
13
|
-
|
14
|
-
def process_http_request
|
15
|
-
# the http request details are available via the following instance variables:
|
16
|
-
# @http_protocol
|
17
|
-
# @http_request_method
|
18
|
-
# @http_cookie
|
19
|
-
# @http_if_none_match
|
20
|
-
# @http_content_type
|
21
|
-
# @http_path_info
|
22
|
-
# @http_request_uri
|
23
|
-
# @http_query_string
|
24
|
-
# @http_post_content
|
25
|
-
# @http_headers
|
26
|
-
response = EM::DelegatedHttpResponse.new(self)
|
27
|
-
response.headers['Refresh'] = '3; url=/'
|
28
|
-
# headers = @http_headers.split("\0").inject({}){|h, s| (s =~ /^([^:]+): (.*)$/ && (h[$1] = $2)); h }
|
29
|
-
|
30
|
-
case @http_request_uri
|
31
|
-
when '/', '/.html'
|
32
|
-
response.content_type 'text/html'
|
33
|
-
server_status(response, "html")
|
34
|
-
when "/.json"
|
35
|
-
response.content_type 'application/json'
|
36
|
-
server_status(response, "json")
|
37
|
-
when "/.txt"
|
38
|
-
response.content_type 'text/plain'
|
39
|
-
server_status(response, "plain")
|
40
|
-
when '/initiate_master_switch'
|
41
|
-
initiate_master_switch(response)
|
42
|
-
when '/brokers'
|
43
|
-
list_brokers(response)
|
44
|
-
else
|
45
|
-
not_found(response)
|
46
|
-
end
|
47
|
-
response.send_response
|
48
|
-
end
|
49
|
-
|
50
|
-
def list_brokers(response)
|
51
|
-
brokers = config_server.config.brokers
|
52
|
-
response.status = 200
|
53
|
-
if @http_headers =~ %r(application/json)
|
54
|
-
response.content_type 'application/json'
|
55
|
-
response.content = brokers.to_json
|
56
|
-
else
|
57
|
-
response.content_type 'text/yaml'
|
58
|
-
response.content = brokers.to_yaml
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
def server_status(response, type)
|
63
|
-
response.status = 200
|
64
|
-
status = config_server.status
|
65
|
-
response.content =
|
66
|
-
case type
|
67
|
-
when "plain"
|
68
|
-
plain_text_response(status)
|
69
|
-
when "json"
|
70
|
-
status.to_json
|
71
|
-
when "html"
|
72
|
-
html_response(status)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def plain_text_response(status)
|
77
|
-
status.keys.sort_by{|k| k.to_s}.reverse.map do |k|
|
78
|
-
name = k.to_s # .split('_').join(" ")
|
79
|
-
if (value = status[k]).is_a?(Array)
|
80
|
-
value = value.empty? ? "none" : value.join(", ")
|
81
|
-
end
|
82
|
-
"#{name}: #{value}"
|
83
|
-
end.join("\n")
|
84
|
-
end
|
85
|
-
|
86
|
-
def html_response(status)
|
87
|
-
b = "<!doctype html>"
|
88
|
-
b << "<html><head><title>Beetle Configuration Server Status</title>#{html_styles(status)}</head>"
|
89
|
-
b << "<body><h1>Beetle Configuration Server Status</h1>"
|
90
|
-
unless status[:redis_master_available?]
|
91
|
-
b << "<form name='masterswitch' method='post' action='/initiate_master_switch'>"
|
92
|
-
b << "Master down! "
|
93
|
-
b << "<a href='javascript: document.masterswitch.submit();'>Initiate master switch</a> "
|
94
|
-
b << "or wait until system performs it automatically."
|
95
|
-
b << "</form>"
|
96
|
-
end
|
97
|
-
b << "<table cellspacing=0>\n"
|
98
|
-
plain_text_response(status).split("\n").compact.each do |row|
|
99
|
-
row =~/(^[^:]+): (.*)$/
|
100
|
-
name, value = $1, $2
|
101
|
-
if value =~ /,/
|
102
|
-
value = "<ul>" << value.split(/\s*,\s*/).map{|s| "<li>#{s}</li>"}.join << "</ul>"
|
103
|
-
end
|
104
|
-
b << "<tr><td>#{name}</td><td>#{value}</td></tr>\n"
|
105
|
-
end
|
106
|
-
b << "</table>"
|
107
|
-
b << "</body></html>"
|
108
|
-
end
|
109
|
-
|
110
|
-
def html_styles(status)
|
111
|
-
warn_color = status[:redis_master_available?] ? "#5780b2" : "#A52A2A"
|
112
|
-
<<"EOS"
|
113
|
-
<style media="screen" type="text/css">
|
114
|
-
html { font: 1.25em/1.5 arial, sans-serif;}
|
115
|
-
body { margin: 1em; }
|
116
|
-
table tr:nth-child(2n+1){ background-color: #ffffff; }
|
117
|
-
td { padding: 0.1em 0.2em; vertical-align: top; }
|
118
|
-
ul { list-style-type: none; margin: 0; padding: 0;}
|
119
|
-
li { }
|
120
|
-
h1 { color: #{warn_color}; margin-bottom: 0.2em;}
|
121
|
-
a:link, a:visited {text-decoration:none; color:#A52A2A;}
|
122
|
-
a:hover, a:active {text-decoration:none; color:#FF0000;}
|
123
|
-
a {
|
124
|
-
padding: 10px; background: #cdcdcd;
|
125
|
-
-moz-border-radius: 5px;
|
126
|
-
border-radius: 5px;
|
127
|
-
-moz-box-shadow: 2px 2px 2px #bbb;
|
128
|
-
-webkit-box-shadow: 2px 2px 2px #bbb;
|
129
|
-
box-shadow: 2px 2px 2px #bbb;
|
130
|
-
}
|
131
|
-
form { font-size: 1em; margin-bottom: 1em; }
|
132
|
-
</style>
|
133
|
-
EOS
|
134
|
-
end
|
135
|
-
|
136
|
-
def initiate_master_switch(response)
|
137
|
-
response.content_type 'text/plain'
|
138
|
-
if config_server.initiate_master_switch
|
139
|
-
response.status = 201
|
140
|
-
response.content = "Master switch initiated"
|
141
|
-
else
|
142
|
-
response.status = 200
|
143
|
-
response.content = "No master switch necessary"
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
def not_found(response)
|
148
|
-
response.content_type 'text/plain'
|
149
|
-
response.status = 404
|
150
|
-
end
|
151
|
-
end
|
152
|
-
end
|