beetle 1.0.3 → 2.0.0rc1

data/lib/beetle/commands/configuration_server.rb

@@ -1,98 +0,0 @@
-require 'optparse'
-require 'daemons'
-require 'beetle'
-
-module Beetle
-  module Commands
-    # Command to start a RedisConfigurationServer daemon.
-    #
-    #   Usage: beetle configuration_server  [options] -- [server options]
-    #
-    #   server options:
-    #           --redis-servers LIST         Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)
-    #           --client-ids LIST            Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)
-    #           --redis-master-file FILE     Write redis master server string to FILE
-    #           --redis-retry-interval SEC   Number of seconds to wait between master checks
-    #           --amqp-servers LIST          AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
-    #           --config-file PATH           Path to an external yaml config file
-    #           --pid-dir DIR                Write pid and log to DIR
-    #       -v, --verbose
-    #       -h, --help                       Show this message
-    #
-    class ConfigurationServer
-      # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
-      def self.execute
-        command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
-
-        opts = OptionParser.new
-        opts.banner = "Usage: beetle configuration_server #{command} [options] -- [server options]"
-        opts.separator ""
-        opts.separator "server options:"
-
-        opts.on("--redis-servers LIST", Array, "Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
-          Beetle.config.redis_servers = val.join(",")
-        end
-
-        opts.on("--client-ids LIST", "Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)") do |val|
-          Beetle.config.redis_configuration_client_ids = val
-        end
-
-        opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
-          Beetle.config.redis_server = val
-        end
-
-        opts.on("--redis-retry-interval SEC", Integer, "Number of seconds to wait between master checks") do |val|
-          Beetle.config.redis_configuration_master_retry_interval = val
-        end
-
-        opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
-          Beetle.config.servers = val
-        end
-
-        opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
-          Beetle.config.config_file = val
-        end
-
-        dir_mode = nil
-        dir = nil
-        opts.on("--pid-dir DIR", String, "Write pid and output to DIR") do |val|
-          dir_mode = :normal
-          dir = val
-        end
-
-        opts.on("-v", "--verbose") do |val|
-          Beetle.config.logger.level = Logger::DEBUG
-        end
-
-        opts.on_tail("-h", "--help", "Show this message") do
-          puts opts
-          exit
-        end
-
-        opts.parse!(app_options)
-
-        if command =~ /start|run/ && Beetle.config.redis_servers.blank?
-          puts opts
-          exit
-        end
-
-        daemon_options = {
-          :log_output => true,
-          :dir_mode   => dir_mode,
-          :dir        => dir,
-          :force      => true
-        }
-
-        Daemons.run_proc("redis_configuration_server", daemon_options) do
-          config_server =  Beetle::RedisConfigurationServer.new
-          Beetle::RedisConfigurationHttpServer.config_server = config_server
-          http_server_port = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
-          EM.run do
-            config_server.start
-            EM.start_server '0.0.0.0', http_server_port, Beetle::RedisConfigurationHttpServer
-          end
-        end
-      end
-    end
-  end
-end

data/lib/beetle/commands/garbage_collect_deduplication_store.rb

@@ -1,52 +0,0 @@
-require 'optparse'
-require 'beetle'
-
-module Beetle
-  module Commands
-    # Command to garbage collect the deduplication store
-    #
-    #   Usage: beetle garbage_collect_deduplication_store [options]
-    #
-    #   options:
-    #       --redis-servers LIST         Required (e.g. 192.168.0.1:6379,192.168.0.2:6379)
-    #       --config-file PATH           Path to an external yaml config file
-    #       -v, --verbose
-    #       -h, --help                   Show this message
-    #
-    class GarbageCollectDeduplicationStore
-      # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
-      def self.execute
-        opts = OptionParser.new
-        opts.banner = "Usage: beetle garbage_collect_deduplication_store [options]"
-        opts.separator ""
-
-        opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
-          Beetle.config.config_file = val
-          Beetle.config.log_file = STDOUT
-        end
-
-        opts.on("--redis-servers LIST", Array, "Comma separted list of redis server:port specs used for GC") do |val|
-          Beetle.config.redis_servers = val.join(",")
-        end
-
-        opts.on("--redis-db N", Integer, "Redis database used for GC") do |val|
-          Beetle.config.redis_db = val.to_i
-        end
-
-        opts.on("-v", "--verbose") do |val|
-          Beetle.config.log_file = STDOUT
-          Beetle.config.logger.level = Logger::DEBUG
-        end
-
-        opts.on_tail("-h", "--help", "Show this message") do
-          puts opts
-          exit
-        end
-
-        opts.parse!(ARGV)
-
-        DeduplicationStore.new.garbage_collect_keys_using_master_and_slave
-      end
-    end
-  end
-end

data/lib/beetle/redis_configuration_client.rb

@@ -1,157 +0,0 @@
-module Beetle
-  # A RedisConfigurationClient is the subordinate part of beetle's
-  # redis failover solution
-  #
-  # An instance of RedisConfigurationClient lives on every server that
-  # hosts message consumers (worker server).
-  #
-  # It is responsible for determining an initial redis master and reacting to redis master
-  # switches initiated by the RedisConfigurationServer.
-  #
-  # It will write the current redis master host:port string to a file specified via a
-  # Configuration, which is then read by DeduplicationStore on redis access.
-  #
-  # Usually started via <tt>beetle configuration_client</tt> command.
-  class RedisConfigurationClient
-    include Logging
-    include RedisMasterFile
-
-    # Set a custom unique id for this instance. Must match an entry in
-    # Configuration#redis_configuration_client_ids.
-    attr_writer :id
-
-    # The current redis master
-    attr_reader :current_master
-
-    # Unique id for this instance (defaults to the fully qualified hostname)
-    def id
-      @id ||= Beetle.hostname
-    end
-
-    def initialize #:nodoc:
-      @current_token = nil
-      MessageDispatcher.configuration_client = self
-    end
-
-    # determinines the initial redis master (if possible), then enters a messaging event
-    # loop, reacting to failover related messages sent by RedisConfigurationServer.
-    def start
-      verify_redis_master_file_string
-      client_started!
-      logger.info "RedisConfigurationClient starting (client id: #{id})"
-      determine_initial_master
-      clear_redis_master_file unless current_master.try(:master?)
-      logger.info "Listening"
-      beetle.listen do
-        EventMachine.add_periodic_timer(Beetle.config.redis_failover_client_heartbeat_interval) do
-          heartbeat!
-        end
-      end
-    end
-
-    # called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
-    def ping(payload)
-      token = payload["token"]
-      logger.info "Received ping message with token '#{token}'"
-      pong! if redeem_token(token)
-    end
-
-    # called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
-    def invalidate(payload)
-      token = payload["token"]
-      logger.info "Received invalidate message with token '#{token}'"
-      invalidate! if redeem_token(token) && !current_master.try(:master?)
-    end
-
-    # called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
-    def reconfigure(payload)
-      server = payload["server"]
-      token = payload["token"]
-      logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
-      return unless redeem_token(token)
-      unless server == read_redis_master_file
-        new_master!(server)
-        write_redis_master_file(server)
-      end
-    end
-
-    # Beetle::Client instance for communication with the RedisConfigurationServer
-    def beetle
-      @beetle ||= build_beetle
-    end
-
-    def config #:nodoc:
-      beetle.config
-    end
-
-    private
-
-    def determine_initial_master
-      if master_file_exists? && server = read_redis_master_file
-        new_master!(server)
-      end
-    end
-
-    def new_master!(server)
-      @current_master = Redis.from_server_string(server, :timeout => 3)
-    end
-
-    def build_beetle
-      system = Beetle.config.system_name
-      Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
-        # messages sent
-        config.message :pong
-        config.message :client_started
-        config.message :client_invalidated
-        config.message :heartbeat
-        # messages received
-        config.message :ping
-        config.message :invalidate
-        config.message :reconfigure
-        # queue setup
-        config.queue   :client, :key => 'ping', :amqp_name => "#{system}_configuration_client_#{id}"
-        config.binding :client, :key => 'invalidate'
-        config.binding :client, :key => 'reconfigure'
-        config.handler :client, MessageDispatcher
-      end
-    end
-
-    def redeem_token(token)
-      @current_token = token if @current_token.nil? || token > @current_token
-      token_valid = token >= @current_token
-      logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
-      token_valid
-    end
-
-    def pong!
-      logger.info "Sending pong message with id '#{id}' and token '#{@current_token}'"
-      beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
-    end
-
-    def client_started!
-      logger.info "Sending client_started message with id '#{id}'"
-      beetle.publish(:client_started, {"id" => id}.to_json)
-    end
-
-    def heartbeat!
-      logger.info "Sending heartbeat message with id '#{id}'"
-      beetle.publish(:heartbeat, {"id" => id}.to_json)
-    end
-
-    def invalidate!
-      @current_master = nil
-      clear_redis_master_file
-      logger.info "Sending client_invalidated message with id '#{id}' and token '#{@current_token}'"
-      beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
-    end
-
-
-    # Dispatches messages from the queue to methods in RedisConfigurationClient
-    class MessageDispatcher < Beetle::Handler #:nodoc:
-      cattr_accessor :configuration_client
-      def process
-        @@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
-      end
-    end
-  end
-end

data/lib/beetle/redis_configuration_http_server.rb

@@ -1,152 +0,0 @@
-require 'evma_httpserver'
-
-module Beetle
-  class RedisConfigurationHttpServer < EM::Connection
-     include EM::HttpServer
-
-     def post_init
-       super
-       no_environment_strings
-     end
-
-    cattr_accessor :config_server
-
-    def process_http_request
-      # the http request details are available via the following instance variables:
-      #   @http_protocol
-      #   @http_request_method
-      #   @http_cookie
-      #   @http_if_none_match
-      #   @http_content_type
-      #   @http_path_info
-      #   @http_request_uri
-      #   @http_query_string
-      #   @http_post_content
-      #   @http_headers
-      response = EM::DelegatedHttpResponse.new(self)
-      response.headers['Refresh'] = '3; url=/'
-      # headers = @http_headers.split("\0").inject({}){|h, s| (s =~ /^([^:]+): (.*)$/ && (h[$1] = $2)); h }
-
-      case @http_request_uri
-      when '/', '/.html'
-        response.content_type 'text/html'
-        server_status(response, "html")
-      when "/.json"
-        response.content_type 'application/json'
-        server_status(response, "json")
-      when "/.txt"
-        response.content_type 'text/plain'
-        server_status(response, "plain")
-      when '/initiate_master_switch'
-        initiate_master_switch(response)
-      when '/brokers'
-        list_brokers(response)
-      else
-        not_found(response)
-      end
-      response.send_response
-    end
-
-    def list_brokers(response)
-      brokers = config_server.config.brokers
-      response.status = 200
-      if @http_headers =~ %r(application/json)
-        response.content_type 'application/json'
-        response.content = brokers.to_json
-      else
-        response.content_type 'text/yaml'
-        response.content = brokers.to_yaml
-      end
-    end
-
-    def server_status(response, type)
-      response.status = 200
-      status = config_server.status
-      response.content =
-        case type
-        when "plain"
-          plain_text_response(status)
-        when "json"
-          status.to_json
-        when "html"
-          html_response(status)
-        end
-    end
-
-    def plain_text_response(status)
-      status.keys.sort_by{|k| k.to_s}.reverse.map do |k|
-        name = k.to_s # .split('_').join(" ")
-        if (value = status[k]).is_a?(Array)
-          value = value.empty? ? "none" : value.join(", ")
-        end
-        "#{name}: #{value}"
-      end.join("\n")
-    end
-
-    def html_response(status)
-      b = "<!doctype html>"
-      b << "<html><head><title>Beetle Configuration Server Status</title>#{html_styles(status)}</head>"
-      b << "<body><h1>Beetle Configuration Server Status</h1>"
-      unless status[:redis_master_available?]
-        b << "<form name='masterswitch' method='post' action='/initiate_master_switch'>"
-        b << "Master down! "
-        b << "<a href='javascript: document.masterswitch.submit();'>Initiate master switch</a> "
-        b << "or wait until system performs it automatically."
-        b << "</form>"
-      end
-      b << "<table cellspacing=0>\n"
-      plain_text_response(status).split("\n").compact.each do |row|
-        row =~/(^[^:]+): (.*)$/
-        name, value = $1, $2
-        if value =~ /,/
-          value = "<ul>" << value.split(/\s*,\s*/).map{|s| "<li>#{s}</li>"}.join << "</ul>"
-        end
-        b << "<tr><td>#{name}</td><td>#{value}</td></tr>\n"
-      end
-      b << "</table>"
-      b << "</body></html>"
-    end
-
-    def html_styles(status)
-      warn_color = status[:redis_master_available?] ? "#5780b2" : "#A52A2A"
-      <<"EOS"
-<style media="screen" type="text/css">
-html { font: 1.25em/1.5 arial, sans-serif;}
-body { margin: 1em; }
-table tr:nth-child(2n+1){ background-color: #ffffff; }
-td { padding: 0.1em 0.2em; vertical-align: top; }
-ul { list-style-type: none; margin: 0; padding: 0;}
-li { }
-h1 { color: #{warn_color}; margin-bottom: 0.2em;}
-a:link, a:visited {text-decoration:none; color:#A52A2A;}
-a:hover, a:active {text-decoration:none; color:#FF0000;}
-a {
-  padding: 10px; background: #cdcdcd;
-  -moz-border-radius: 5px;
-   border-radius: 5px;
-  -moz-box-shadow: 2px 2px 2px #bbb;
-  -webkit-box-shadow: 2px 2px 2px #bbb;
-  box-shadow: 2px 2px 2px #bbb;
-}
-form { font-size: 1em; margin-bottom: 1em; }
-</style>
-EOS
-    end
-
-    def initiate_master_switch(response)
-      response.content_type 'text/plain'
-      if config_server.initiate_master_switch
-        response.status = 201
-        response.content = "Master switch initiated"
-      else
-        response.status = 200
-        response.content = "No master switch necessary"
-      end
-    end
-
-    def not_found(response)
-      response.content_type 'text/plain'
-      response.status = 404
-    end
-  end
-end