beetle 1.0.3 → 2.0.0rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,98 +0,0 @@
1
- require 'optparse'
2
- require 'daemons'
3
- require 'beetle'
4
-
5
- module Beetle
6
- module Commands
7
- # Command to start a RedisConfigurationServer daemon.
8
- #
9
- # Usage: beetle configuration_server [options] -- [server options]
10
- #
11
- # server options:
12
- # --redis-servers LIST Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)
13
- # --client-ids LIST Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)
14
- # --redis-master-file FILE Write redis master server string to FILE
15
- # --redis-retry-interval SEC Number of seconds to wait between master checks
16
- # --amqp-servers LIST AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
17
- # --config-file PATH Path to an external yaml config file
18
- # --pid-dir DIR Write pid and log to DIR
19
- # -v, --verbose
20
- # -h, --help Show this message
21
- #
22
- class ConfigurationServer
23
- # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
24
- def self.execute
25
- command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
26
-
27
- opts = OptionParser.new
28
- opts.banner = "Usage: beetle configuration_server #{command} [options] -- [server options]"
29
- opts.separator ""
30
- opts.separator "server options:"
31
-
32
- opts.on("--redis-servers LIST", Array, "Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
33
- Beetle.config.redis_servers = val.join(",")
34
- end
35
-
36
- opts.on("--client-ids LIST", "Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)") do |val|
37
- Beetle.config.redis_configuration_client_ids = val
38
- end
39
-
40
- opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
41
- Beetle.config.redis_server = val
42
- end
43
-
44
- opts.on("--redis-retry-interval SEC", Integer, "Number of seconds to wait between master checks") do |val|
45
- Beetle.config.redis_configuration_master_retry_interval = val
46
- end
47
-
48
- opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
49
- Beetle.config.servers = val
50
- end
51
-
52
- opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
53
- Beetle.config.config_file = val
54
- end
55
-
56
- dir_mode = nil
57
- dir = nil
58
- opts.on("--pid-dir DIR", String, "Write pid and output to DIR") do |val|
59
- dir_mode = :normal
60
- dir = val
61
- end
62
-
63
- opts.on("-v", "--verbose") do |val|
64
- Beetle.config.logger.level = Logger::DEBUG
65
- end
66
-
67
- opts.on_tail("-h", "--help", "Show this message") do
68
- puts opts
69
- exit
70
- end
71
-
72
- opts.parse!(app_options)
73
-
74
- if command =~ /start|run/ && Beetle.config.redis_servers.blank?
75
- puts opts
76
- exit
77
- end
78
-
79
- daemon_options = {
80
- :log_output => true,
81
- :dir_mode => dir_mode,
82
- :dir => dir,
83
- :force => true
84
- }
85
-
86
- Daemons.run_proc("redis_configuration_server", daemon_options) do
87
- config_server = Beetle::RedisConfigurationServer.new
88
- Beetle::RedisConfigurationHttpServer.config_server = config_server
89
- http_server_port = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
90
- EM.run do
91
- config_server.start
92
- EM.start_server '0.0.0.0', http_server_port, Beetle::RedisConfigurationHttpServer
93
- end
94
- end
95
- end
96
- end
97
- end
98
- end
@@ -1,52 +0,0 @@
1
- require 'optparse'
2
- require 'beetle'
3
-
4
- module Beetle
5
- module Commands
6
- # Command to garbage collect the deduplication store
7
- #
8
- # Usage: beetle garbage_collect_deduplication_store [options]
9
- #
10
- # options:
11
- # --redis-servers LIST Required (e.g. 192.168.0.1:6379,192.168.0.2:6379)
12
- # --config-file PATH Path to an external yaml config file
13
- # -v, --verbose
14
- # -h, --help Show this message
15
- #
16
- class GarbageCollectDeduplicationStore
17
- # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
18
- def self.execute
19
- opts = OptionParser.new
20
- opts.banner = "Usage: beetle garbage_collect_deduplication_store [options]"
21
- opts.separator ""
22
-
23
- opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
24
- Beetle.config.config_file = val
25
- Beetle.config.log_file = STDOUT
26
- end
27
-
28
- opts.on("--redis-servers LIST", Array, "Comma separted list of redis server:port specs used for GC") do |val|
29
- Beetle.config.redis_servers = val.join(",")
30
- end
31
-
32
- opts.on("--redis-db N", Integer, "Redis database used for GC") do |val|
33
- Beetle.config.redis_db = val.to_i
34
- end
35
-
36
- opts.on("-v", "--verbose") do |val|
37
- Beetle.config.log_file = STDOUT
38
- Beetle.config.logger.level = Logger::DEBUG
39
- end
40
-
41
- opts.on_tail("-h", "--help", "Show this message") do
42
- puts opts
43
- exit
44
- end
45
-
46
- opts.parse!(ARGV)
47
-
48
- DeduplicationStore.new.garbage_collect_keys_using_master_and_slave
49
- end
50
- end
51
- end
52
- end
@@ -1,157 +0,0 @@
1
- module Beetle
2
- # A RedisConfigurationClient is the subordinate part of beetle's
3
- # redis failover solution
4
- #
5
- # An instance of RedisConfigurationClient lives on every server that
6
- # hosts message consumers (worker server).
7
- #
8
- # It is responsible for determining an initial redis master and reacting to redis master
9
- # switches initiated by the RedisConfigurationServer.
10
- #
11
- # It will write the current redis master host:port string to a file specified via a
12
- # Configuration, which is then read by DeduplicationStore on redis access.
13
- #
14
- # Usually started via <tt>beetle configuration_client</tt> command.
15
- class RedisConfigurationClient
16
- include Logging
17
- include RedisMasterFile
18
-
19
- # Set a custom unique id for this instance. Must match an entry in
20
- # Configuration#redis_configuration_client_ids.
21
- attr_writer :id
22
-
23
- # The current redis master
24
- attr_reader :current_master
25
-
26
- # Unique id for this instance (defaults to the fully qualified hostname)
27
- def id
28
- @id ||= Beetle.hostname
29
- end
30
-
31
- def initialize #:nodoc:
32
- @current_token = nil
33
- MessageDispatcher.configuration_client = self
34
- end
35
-
36
- # determinines the initial redis master (if possible), then enters a messaging event
37
- # loop, reacting to failover related messages sent by RedisConfigurationServer.
38
- def start
39
- verify_redis_master_file_string
40
- client_started!
41
- logger.info "RedisConfigurationClient starting (client id: #{id})"
42
- determine_initial_master
43
- clear_redis_master_file unless current_master.try(:master?)
44
- logger.info "Listening"
45
- beetle.listen do
46
- EventMachine.add_periodic_timer(Beetle.config.redis_failover_client_heartbeat_interval) do
47
- heartbeat!
48
- end
49
- end
50
- end
51
-
52
- # called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
53
- def ping(payload)
54
- token = payload["token"]
55
- logger.info "Received ping message with token '#{token}'"
56
- pong! if redeem_token(token)
57
- end
58
-
59
- # called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
60
- def invalidate(payload)
61
- token = payload["token"]
62
- logger.info "Received invalidate message with token '#{token}'"
63
- invalidate! if redeem_token(token) && !current_master.try(:master?)
64
- end
65
-
66
- # called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
67
- def reconfigure(payload)
68
- server = payload["server"]
69
- token = payload["token"]
70
- logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
71
- return unless redeem_token(token)
72
- unless server == read_redis_master_file
73
- new_master!(server)
74
- write_redis_master_file(server)
75
- end
76
- end
77
-
78
- # Beetle::Client instance for communication with the RedisConfigurationServer
79
- def beetle
80
- @beetle ||= build_beetle
81
- end
82
-
83
- def config #:nodoc:
84
- beetle.config
85
- end
86
-
87
- private
88
-
89
- def determine_initial_master
90
- if master_file_exists? && server = read_redis_master_file
91
- new_master!(server)
92
- end
93
- end
94
-
95
- def new_master!(server)
96
- @current_master = Redis.from_server_string(server, :timeout => 3)
97
- end
98
-
99
- def build_beetle
100
- system = Beetle.config.system_name
101
- Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
102
- # messages sent
103
- config.message :pong
104
- config.message :client_started
105
- config.message :client_invalidated
106
- config.message :heartbeat
107
- # messages received
108
- config.message :ping
109
- config.message :invalidate
110
- config.message :reconfigure
111
- # queue setup
112
- config.queue :client, :key => 'ping', :amqp_name => "#{system}_configuration_client_#{id}"
113
- config.binding :client, :key => 'invalidate'
114
- config.binding :client, :key => 'reconfigure'
115
- config.handler :client, MessageDispatcher
116
- end
117
- end
118
-
119
- def redeem_token(token)
120
- @current_token = token if @current_token.nil? || token > @current_token
121
- token_valid = token >= @current_token
122
- logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
123
- token_valid
124
- end
125
-
126
- def pong!
127
- logger.info "Sending pong message with id '#{id}' and token '#{@current_token}'"
128
- beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
129
- end
130
-
131
- def client_started!
132
- logger.info "Sending client_started message with id '#{id}'"
133
- beetle.publish(:client_started, {"id" => id}.to_json)
134
- end
135
-
136
- def heartbeat!
137
- logger.info "Sending heartbeat message with id '#{id}'"
138
- beetle.publish(:heartbeat, {"id" => id}.to_json)
139
- end
140
-
141
- def invalidate!
142
- @current_master = nil
143
- clear_redis_master_file
144
- logger.info "Sending client_invalidated message with id '#{id}' and token '#{@current_token}'"
145
- beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
146
- end
147
-
148
-
149
- # Dispatches messages from the queue to methods in RedisConfigurationClient
150
- class MessageDispatcher < Beetle::Handler #:nodoc:
151
- cattr_accessor :configuration_client
152
- def process
153
- @@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
154
- end
155
- end
156
- end
157
- end
@@ -1,152 +0,0 @@
1
- require 'evma_httpserver'
2
-
3
- module Beetle
4
- class RedisConfigurationHttpServer < EM::Connection
5
- include EM::HttpServer
6
-
7
- def post_init
8
- super
9
- no_environment_strings
10
- end
11
-
12
- cattr_accessor :config_server
13
-
14
- def process_http_request
15
- # the http request details are available via the following instance variables:
16
- # @http_protocol
17
- # @http_request_method
18
- # @http_cookie
19
- # @http_if_none_match
20
- # @http_content_type
21
- # @http_path_info
22
- # @http_request_uri
23
- # @http_query_string
24
- # @http_post_content
25
- # @http_headers
26
- response = EM::DelegatedHttpResponse.new(self)
27
- response.headers['Refresh'] = '3; url=/'
28
- # headers = @http_headers.split("\0").inject({}){|h, s| (s =~ /^([^:]+): (.*)$/ && (h[$1] = $2)); h }
29
-
30
- case @http_request_uri
31
- when '/', '/.html'
32
- response.content_type 'text/html'
33
- server_status(response, "html")
34
- when "/.json"
35
- response.content_type 'application/json'
36
- server_status(response, "json")
37
- when "/.txt"
38
- response.content_type 'text/plain'
39
- server_status(response, "plain")
40
- when '/initiate_master_switch'
41
- initiate_master_switch(response)
42
- when '/brokers'
43
- list_brokers(response)
44
- else
45
- not_found(response)
46
- end
47
- response.send_response
48
- end
49
-
50
- def list_brokers(response)
51
- brokers = config_server.config.brokers
52
- response.status = 200
53
- if @http_headers =~ %r(application/json)
54
- response.content_type 'application/json'
55
- response.content = brokers.to_json
56
- else
57
- response.content_type 'text/yaml'
58
- response.content = brokers.to_yaml
59
- end
60
- end
61
-
62
- def server_status(response, type)
63
- response.status = 200
64
- status = config_server.status
65
- response.content =
66
- case type
67
- when "plain"
68
- plain_text_response(status)
69
- when "json"
70
- status.to_json
71
- when "html"
72
- html_response(status)
73
- end
74
- end
75
-
76
- def plain_text_response(status)
77
- status.keys.sort_by{|k| k.to_s}.reverse.map do |k|
78
- name = k.to_s # .split('_').join(" ")
79
- if (value = status[k]).is_a?(Array)
80
- value = value.empty? ? "none" : value.join(", ")
81
- end
82
- "#{name}: #{value}"
83
- end.join("\n")
84
- end
85
-
86
- def html_response(status)
87
- b = "<!doctype html>"
88
- b << "<html><head><title>Beetle Configuration Server Status</title>#{html_styles(status)}</head>"
89
- b << "<body><h1>Beetle Configuration Server Status</h1>"
90
- unless status[:redis_master_available?]
91
- b << "<form name='masterswitch' method='post' action='/initiate_master_switch'>"
92
- b << "Master down! "
93
- b << "<a href='javascript: document.masterswitch.submit();'>Initiate master switch</a> "
94
- b << "or wait until system performs it automatically."
95
- b << "</form>"
96
- end
97
- b << "<table cellspacing=0>\n"
98
- plain_text_response(status).split("\n").compact.each do |row|
99
- row =~/(^[^:]+): (.*)$/
100
- name, value = $1, $2
101
- if value =~ /,/
102
- value = "<ul>" << value.split(/\s*,\s*/).map{|s| "<li>#{s}</li>"}.join << "</ul>"
103
- end
104
- b << "<tr><td>#{name}</td><td>#{value}</td></tr>\n"
105
- end
106
- b << "</table>"
107
- b << "</body></html>"
108
- end
109
-
110
- def html_styles(status)
111
- warn_color = status[:redis_master_available?] ? "#5780b2" : "#A52A2A"
112
- <<"EOS"
113
- <style media="screen" type="text/css">
114
- html { font: 1.25em/1.5 arial, sans-serif;}
115
- body { margin: 1em; }
116
- table tr:nth-child(2n+1){ background-color: #ffffff; }
117
- td { padding: 0.1em 0.2em; vertical-align: top; }
118
- ul { list-style-type: none; margin: 0; padding: 0;}
119
- li { }
120
- h1 { color: #{warn_color}; margin-bottom: 0.2em;}
121
- a:link, a:visited {text-decoration:none; color:#A52A2A;}
122
- a:hover, a:active {text-decoration:none; color:#FF0000;}
123
- a {
124
- padding: 10px; background: #cdcdcd;
125
- -moz-border-radius: 5px;
126
- border-radius: 5px;
127
- -moz-box-shadow: 2px 2px 2px #bbb;
128
- -webkit-box-shadow: 2px 2px 2px #bbb;
129
- box-shadow: 2px 2px 2px #bbb;
130
- }
131
- form { font-size: 1em; margin-bottom: 1em; }
132
- </style>
133
- EOS
134
- end
135
-
136
- def initiate_master_switch(response)
137
- response.content_type 'text/plain'
138
- if config_server.initiate_master_switch
139
- response.status = 201
140
- response.content = "Master switch initiated"
141
- else
142
- response.status = 200
143
- response.content = "No master switch necessary"
144
- end
145
- end
146
-
147
- def not_found(response)
148
- response.content_type 'text/plain'
149
- response.status = 404
150
- end
151
- end
152
- end