beetle 1.0.4 → 2.0.0rc1

data/lib/beetle/redis_configuration_server.rb

@@ -1,438 +0,0 @@
-module Beetle
-  # A RedisConfigurationServer is the supervisor part of beetle's
-  # redis failover solution.
-  #
-  # A single instance of RedisConfigurationServer works as a supervisor for
-  # several RedisConfigurationClient instances. It is responsible for watching
-  # the redis master and electing and publishing a new master in case of failure.
-  #
-  # It will make sure that all configured RedisConfigurationClient instances
-  # do not use the old master anymore before making a switch, to prevent
-  # inconsistent data.
-  #
-  # Usually started via <tt>beetle configuration_server</tt> command.
-  class RedisConfigurationServer
-    include Logging
-    include RedisMasterFile
-
-    # the current redis master
-    attr_reader :current_master
-
-    # the current token used to detect correct message order
-    attr_reader :current_token
-
-    # the list of known client ids
-    attr_reader :client_ids
-
-    # the list of client ids we have received but we don't know about
-    attr_reader :unknown_client_ids
-
-    # clients last seen time stamps
-    attr_reader :clients_last_seen
-
-    # clients are presumed dead after this many seconds
-    attr_accessor :client_dead_threshold
-
-    def initialize #:nodoc:
-      @client_ids = Set.new(config.redis_configuration_client_ids.split(","))
-      @unknown_client_ids = Set.new
-      @clients_last_seen = {}
-      @current_token = (Time.now.to_f * 1000).to_i
-      @client_pong_ids_received = Set.new
-      @client_invalidated_ids_received = Set.new
-      @client_dead_threshold = config.redis_failover_client_dead_interval
-      MessageDispatcher.configuration_server = self
-    end
-
-    # Redis system status information (an instance of class RedisServerInfo)
-    def redis
-      @redis ||= RedisServerInfo.new(config, :timeout => 3)
-    end
-
-    # Beetle::Client instance for communication with the RedisConfigurationServer
-    def beetle
-      @beetle ||= build_beetle
-    end
-
-    def config #:nodoc:
-      beetle.config
-    end
-
-    # returns a hash describing the current server status
-    def status
-      {
-        :beetle_version => Beetle::VERSION,
-        :configured_brokers => config.servers.split(/\s*,\s*/),
-        :configured_client_ids => client_ids.to_a.sort,
-        :configured_redis_servers => config.redis_servers.split(/\s*,\s*/),
-        :redis_master => current_master.try(:server).to_s,
-        :redis_master_available? => master_available?,
-        :redis_slaves_available => available_slaves.map(&:server),
-        :switch_in_progress => paused?,
-        :unknown_client_ids => unknown_client_ids.to_a,
-        :unresponsive_clients => unresponsive_clients.map{|c,t| "#{c}:#{t.to_i}"},
-        :unseen_client_ids => unseen_client_ids
-      }
-    end
-
-    # return array of client ids which never sent a ping
-    def unseen_client_ids
-      (client_ids - clients_last_seen.keys.to_set).to_a.sort
-    end
-
-    # returns an array of [client_id, silent time in seconds] which haven't sent
-    # a message for more than <tt>client_dead_threshold</tt> seconds, sorted by
-    # silent time in descending order.
-    def unresponsive_clients
-      now = Time.now
-      threshold = now - client_dead_threshold
-      clients_last_seen.select{|_,t| t < threshold}.map{|c,t| [c, now - t]}.sort_by{|_,t| t}.reverse
-    end
-
-    # start watching redis
-    def start
-      verify_redis_master_file_string
-      check_redis_configuration
-      redis.refresh
-      determine_initial_master
-      log_start
-      beetle.listen do
-        master_watcher.watch
-      end
-    end
-
-    # test if redis is currently being watched
-    def paused?
-      master_watcher.paused?
-    end
-
-    # called by the message dispatcher when a "pong" message from a RedisConfigurationClient is received
-    def pong(payload)
-      id = payload["id"]
-      client_seen(id)
-      token = payload["token"]
-      return unless validate_pong_client_id(id)
-      logger.info "Received pong message from id '#{id}' with token '#{token}'"
-      return unless redeem_token(token)
-      @client_pong_ids_received << id
-      if all_client_pong_ids_received?
-        logger.info "All client pong messages received"
-        @available_timer.cancel if @available_timer
-        invalidate_current_master
-      end
-    end
-
-    # called by the message dispatcher when a "client_started" message from a RedisConfigurationClient is received
-    def client_started(payload)
-      id = payload["id"]
-      client_seen(id)
-      if client_id_valid?(id)
-        logger.info "Received client_started message from id '#{id}'"
-      else
-        add_unknown_client_id(id)
-        msg = "Received client_started message from unknown id '#{id}'"
-        logger.error msg
-        beetle.publish(:system_notification, {"message" => msg}.to_json)
-      end
-    end
-
-    # called by the message dispatcher when a "heartbeat" message from a RedisConfigurationClient is received
-    def heartbeat(payload)
-      id = payload["id"]
-      client_seen(id)
-      if client_id_valid?(id)
-        logger.info "Received heartbeat message from id '#{id}'"
-      else
-        add_unknown_client_id(id)
-        msg = "Received heartbeat message from unknown id '#{id}'"
-        logger.error msg
-        beetle.publish(:system_notification, {"message" => msg}.to_json)
-      end
-    end
-
-    # called by the message dispatcher when a "client_invalidated" message from a RedisConfigurationClient is received
-    def client_invalidated(payload)
-      id = payload["id"]
-      client_seen(id)
-      token = payload["token"]
-      add_unknown_client_id(id) unless client_id_valid?(id)
-      logger.info "Received client_invalidated message from id '#{id}' with token '#{token}'"
-      return unless redeem_token(token)
-      @client_invalidated_ids_received << id
-      if all_client_invalidated_ids_received?
-        logger.debug "All client invalidated messages received"
-        @invalidate_timer.cancel if @invalidate_timer
-        switch_master
-      end
-    end
-
-    # called from RedisWatcher when watched redis becomes unavailable
-    def master_unavailable!
-      msg = "Redis master '#{current_master.server}' not available"
-      master_watcher.pause
-      logger.warn(msg)
-      beetle.publish(:system_notification, {"message" => msg}.to_json)
-
-      if @client_ids.empty?
-        switch_master
-      else
-        start_invalidation
-      end
-    end
-
-    # called from RedisWatcher when watched redis is available
-    def master_available!
-      publish_master(current_master)
-      configure_slaves(current_master)
-    end
-
-    # check whether the current master is still online and still a master
-    def master_available?
-      redis.masters.include?(current_master)
-    end
-
-    # list of available redis slaves
-    def available_slaves
-      redis.slaves
-    end
-
-    # initiate a master switch if the current master is not available and no switch is in progress
-    def initiate_master_switch
-      redis.refresh
-      available, switch_in_progress = master_available?, paused?
-      logger.debug "Initiating master switch: already in progress = #{switch_in_progress}"
-      unless available || switch_in_progress
-        master_unavailable!
-      end
-      !available || switch_in_progress
-    end
-
-    private
-
-    # prevent memory overflows caused by evil or buggy clients
-    MAX_UNKNOWN_CLIENT_IDS = 20
-
-    def add_unknown_client_id(id)
-      ids = @unknown_client_ids
-      while ids.size >= MAX_UNKNOWN_CLIENT_IDS
-        old_id = ids.first
-        ids.delete(old_id)
-        clients_last_seen.delete(old_id)
-      end
-      ids << id
-    end
-
-    def client_seen(client_id)
-      clients_last_seen[client_id] = Time.now
-    end
-
-    def check_redis_configuration
-      raise ConfigurationError.new("Redis failover needs two or more redis servers") if redis.instances.size < 2
-    end
-
-    def log_start
-      logger.info "RedisConfigurationServer starting"
-      logger.info "AMQP servers  : #{config.servers}"
-      logger.info "Client ids    : #{config.redis_configuration_client_ids}"
-      logger.info "Redis servers : #{config.redis_servers}"
-      logger.info "Redis master  : #{current_master.server}"
-    end
-
-    def build_beetle
-      system = Beetle.config.system_name
-      Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
-        # messages sent
-        config.message :ping
-        config.message :invalidate
-        config.message :reconfigure
-        config.message :system_notification
-        # messages received
-        config.message :client_started
-        config.message :pong
-        config.message :client_invalidated
-        config.message :heartbeat
-        # queue setup
-        config.queue   :server, :key => 'pong', :amqp_name => "#{system}_configuration_server"
-        config.binding :server, :key => 'client_started'
-        config.binding :server, :key => 'client_invalidated'
-        config.binding :server, :key => 'heartbeat'
-        config.handler :server, MessageDispatcher
-      end
-    end
-
-    def master_watcher
-      @master_watcher ||= RedisWatcher.new(self)
-    end
-
-    def determine_initial_master
-      if master_file_exists? && @current_master = redis_master_from_master_file
-        if redis.slaves.include?(current_master)
-          master_unavailable!
-        elsif redis.unknowns.include?(current_master)
-          master_unavailable!
-        end
-      else
-        write_redis_master_file(current_master.server) if @current_master = redis.auto_detect_master
-      end
-      current_master or raise NoRedisMaster.new("Failed to determine initial redis master")
-    end
-
-    def determine_new_master
-      redis.unknowns.include?(current_master) ? redis.slaves_of(current_master).first : current_master
-    end
-
-    def validate_pong_client_id(client_id)
-      unless known_client = client_id_valid?(client_id)
-        add_unknown_client_id(client_id)
-        msg = "Received pong message from unknown id '#{client_id}'"
-        logger.error msg
-        logger.info "Sending system_notification message with text: #{msg}"
-        beetle.publish(:system_notification, {"message" => msg}.to_json)
-      end
-      known_client
-    end
-
-    def client_id_valid?(client_id)
-      @client_ids.include?(client_id)
-    end
-
-    def redeem_token(token)
-      valid_token = token == @current_token
-      logger.info "Ignored message (token was '#{token.inspect}', but expected '#{@current_token.inspect}')" unless valid_token
-      valid_token
-    end
-
-    def start_invalidation
-      @client_pong_ids_received.clear
-      @client_invalidated_ids_received.clear
-      check_all_clients_available
-    end
-
-    def check_all_clients_available
-      generate_new_token
-      logger.info "Sending ping message with token '#{@current_token}'"
-      beetle.publish(:ping, payload_with_current_token)
-      @available_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
-    end
-
-    def invalidate_current_master
-      generate_new_token
-      logger.info "Sending invalidate message with token '#{@current_token}'"
-      beetle.publish(:invalidate, payload_with_current_token)
-      @invalidate_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
-    end
-
-    def cancel_invalidation
-      logger.warn "Redis master invalidation cancelled: 'pong' received from '#{@client_pong_ids_received.to_a.join(',')}', 'client_invalidated' received from '#{@client_invalidated_ids_received.to_a.join(',')}'"
-      generate_new_token
-      master_watcher.continue
-    end
-
-    def generate_new_token
-      @current_token += 1
-    end
-
-    def payload_with_current_token(message = {})
-      message["token"] = @current_token
-      message.to_json
-    end
-
-    def all_client_pong_ids_received?
-      @client_ids == @client_pong_ids_received
-    end
-
-    def all_client_invalidated_ids_received?
-      @client_ids == @client_invalidated_ids_received
-    end
-
-    def switch_master
-      if new_master = determine_new_master
-        msg = "Setting redis master to '#{new_master.server}' (was '#{current_master.server}')"
-        logger.warn msg
-        logger.info "Sending system_notification message with text: #{msg}"
-        beetle.publish(:system_notification, {"message" => msg}.to_json)
-
-        new_master.master!
-        write_redis_master_file(new_master.server)
-        @current_master = new_master
-      else
-        msg = "Redis master could not be switched, no slave available to become new master, promoting old master"
-        logger.error msg
-        logger.info "Sending system_notification message with text: #{msg}"
-        beetle.publish(:system_notification, {"message" => msg}.to_json)
-      end
-
-      publish_master(current_master)
-      master_watcher.continue
-    end
-
-    def publish_master(master)
-      logger.info "Sending reconfigure message with server '#{master.server}'"
-      beetle.publish(:reconfigure, payload_with_current_token({"server" => master.server}))
-    end
-
-    def configure_slaves(master)
-      (redis.masters-[master]).each do |r|
-        logger.info "Reconfiguring '#{r.server}' as a slave of '#{master.server}'"
-        r.slave_of!(master.host, master.port)
-      end
-    end
-
-    # Periodically checks a redis server for availability
-    class RedisWatcher #:nodoc:
-      include Logging
-
-      def initialize(configuration_server)
-        @configuration_server = configuration_server
-        @retries = 0
-        @paused = true
-        @master_retry_interval = configuration_server.config.redis_configuration_master_retry_interval
-        @master_retries = configuration_server.config.redis_configuration_master_retries
-      end
-
-      def pause
-        logger.info "Pause checking availability of redis servers"
-        @watch_timer.cancel if @watch_timer
-        @watch_timer = nil
-        @paused = true
-      end
-
-      def watch
-        @watch_timer ||=
-          begin
-            logger.info "Start watching redis servers every #{@master_retry_interval} seconds"
-            EventMachine::add_periodic_timer(@master_retry_interval) { check_availability }
-          end
-        @paused = false
-      end
-      alias continue watch
-
-      def paused?
-        @paused
-      end
-
-      private
-
-      def check_availability
-        @configuration_server.redis.refresh
-        if @configuration_server.master_available?
-          @configuration_server.master_available!
-        else
-          logger.warn "Redis master not available! (Retries left: #{@master_retries - (@retries + 1)})"
-          if (@retries+=1) >= @master_retries
-            @retries = 0
-            @configuration_server.master_unavailable!
-          end
-        end
-      end
-    end
-
-    # Dispatches messages from the queue to methods in RedisConfigurationServer
-    class MessageDispatcher < Beetle::Handler #:nodoc:
-      cattr_accessor :configuration_server
-      def process
-        @@configuration_server.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
-      end
-    end
-  end
-end

data/lib/beetle/redis_server_info.rb

@@ -1,66 +0,0 @@
-module Beetle
-  # class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
-  class RedisServerInfo
-    include Logging
-
-    def initialize(config, options) #:nodoc:
-      @config = config
-      @options = options
-      reset
-    end
-
-    # all configured redis servers
-    def instances
-      @instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
-    end
-
-    # fetches the server from the instances whith the given <tt>server</tt> string
-    def find(server)
-      instances.find{|r| r.server == server}
-    end
-
-    # refresh connectivity/role information
-    def refresh
-      logger.debug "Updating redis server info"
-      reset
-      instances.each {|r| @server_info[r.role] << r}
-    end
-
-    # subset of instances which are masters
-    def masters
-      @server_info["master"]
-    end
-
-    # subset of instances which are slaves
-    def slaves
-      @server_info["slave"]
-    end
-
-    # subset of instances which are not reachable
-    def unknowns
-      @server_info["unknown"]
-    end
-
-    # subset of instances which are set up as slaves of the given <tt>master</tt>
-    def slaves_of(master)
-      slaves.select{|r| r.slave_of?(master.host, master.port)}
-    end
-
-    # determine a master if we have one master and all other servers are slaves
-    def auto_detect_master
-      return nil unless master_and_slaves_reachable?
-      masters.first
-    end
-
-    # check whether all redis servers are up and configured
-    def master_and_slaves_reachable?
-      masters.size == 1 && slaves.size == instances.size - 1
-    end
-
-    private
-
-    def reset
-      @server_info = Hash.new {|h,k| h[k]= []}
-    end
-  end
-end

data/test/beetle/redis_configuration_client_test.rb

@@ -1,118 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
-
-module Beetle
-  class RedisConfigurationClientTest < Minitest::Test
-    def setup
-      Beetle.config.redis_servers = "redis:0,redis:1"
-      @client = RedisConfigurationClient.new
-      @client.stubs(:touch_master_file)
-      @client.stubs(:verify_redis_master_file_string)
-    end
-
-    test "should send a client_started message when started" do
-      @client.stubs(:clear_redis_master_file)
-      @client.beetle.expects(:publish).with(:client_started, {:id => @client.id}.to_json)
-      Client.any_instance.stubs(:listen)
-      @client.start
-    end
-
-    test "should install a periodic timer which sends a heartbeat message" do
-      @client.stubs(:clear_redis_master_file)
-      @client.beetle.expects(:publish).with(:client_started, {:id => @client.id}.to_json)
-      @client.beetle.expects(:listen).yields
-      EventMachine.expects(:add_periodic_timer).with(Beetle.config.redis_failover_client_heartbeat_interval).yields
-      @client.beetle.expects(:publish).with(:heartbeat, {:id => @client.id}.to_json)
-      @client.start
-    end
-
-    test "config should return the beetle config" do
-      assert_equal Beetle.config, @client.config
-    end
-
-    test "ping message should answer with pong" do
-      @client.expects(:pong!)
-      @client.ping("token" => 1)
-    end
-
-    test "pong should publish a pong message" do
-      @client.beetle.expects(:publish)
-      @client.send(:pong!)
-    end
-
-    test "invalidation should send an invalidation message and clear the redis master file" do
-      @client.expects(:clear_redis_master_file)
-      @client.beetle.expects(:publish).with(:client_invalidated, anything)
-      @client.send(:invalidate!)
-    end
-
-    test "should ignore outdated invalidate messages" do
-      new_payload = {"token" => 2}
-      old_payload = {"token" => 1}
-
-      @client.expects(:invalidate!).once
-
-      @client.invalidate(new_payload)
-      @client.invalidate(old_payload)
-    end
-
-    test "should ignore invalidate messages when current master is still a master" do
-      @client.instance_variable_set :@current_master, stub(:master? => true)
-      @client.expects(:invalidate!).never
-      @client.invalidate("token" => 1)
-    end
-
-    test "should ignore outdated reconfigure messages" do
-      new_payload = {"token" => 2, "server" => "master:2"}
-      old_payload = {"token" => 1, "server" => "master:1"}
-      @client.stubs(:read_redis_master_file).returns("")
-
-      @client.expects(:write_redis_master_file).once
-
-      @client.reconfigure(new_payload)
-      @client.reconfigure(old_payload)
-    end
-
-    test "should clear redis master file if redis from master file is slave" do
-      @client.stubs(:redis_master_from_master_file).returns(stub(:master? => false))
-      Beetle::Client.any_instance.stubs(:publish)
-      @client.expects(:clear_redis_master_file)
-      @client.expects(:client_started!)
-      Client.any_instance.stubs(:listen)
-      @client.start
-    end
-
-    test "should clear redis master file if redis from master file is not available" do
-      @client.stubs(:redis_master_from_master_file).returns(nil)
-      Beetle::Client.any_instance.stubs(:publish)
-      @client.expects(:clear_redis_master_file)
-      @client.expects(:client_started!)
-      Client.any_instance.stubs(:listen)
-      @client.start
-    end
-
-    test "the dispatcher should just forward messages to the client" do
-      dispatcher_class = RedisConfigurationClient.class_eval "MessageDispatcher"
-      dispatcher_class.configuration_client = @client
-      dispatcher = dispatcher_class.new
-      payload = {"token" => 1}
-      dispatcher.stubs(:message).returns(stub(:data => payload.to_json, :header => stub(:routing_key=> "ping")))
-      @client.expects(:ping).with(payload)
-      dispatcher.send(:process)
-    end
-
-    test "determine_initial_master should return nil if there is no file" do
-      @client.expects(:master_file_exists?).returns(false)
-      assert_nil @client.send(:determine_initial_master)
-      assert_nil @client.current_master
-    end
-
-    test "determine_initial_master should instantiate a new redis if there is a file with content" do
-      @client.expects(:master_file_exists?).returns(true)
-      @client.expects(:read_redis_master_file).returns("localhost:6379")
-      master = @client.send(:determine_initial_master)
-      assert_equal "master", master.role
-      assert_equal master, @client.current_master
-    end
-
-  end
-end