beetle 0.1 → 0.2.0

data/lib/beetle/redis_configuration_server.rb

@@ -0,0 +1,301 @@
+module Beetle
+  # A RedisConfigurationServer is the supervisor part of beetle's
+  # redis failover solution
+  #
+  # An single instance of RedisConfigurationServer works as a supervisor for
+  # several RedisConfigurationClient instances. It is responsible for watching
+  # the redis master and electing and publishing a new master in case of failure.
+  #
+  # It will make sure that all configured RedisConfigurationClient instances
+  # do not use the old master anymore before making a switch, to prevent
+  # inconsistent data.
+  #
+  # Usually started via <tt>beetle configuration_server</tt> command.
+  class RedisConfigurationServer
+    include Logging
+    include RedisMasterFile
+
+    # the current redis master
+    attr_reader :current_master
+
+    # the current token used to detect correct message order
+    attr_reader :current_token
+
+    def initialize #:nodoc:
+      @client_ids = Set.new(config.redis_configuration_client_ids.split(","))
+      @current_token = (Time.now.to_f * 1000).to_i
+      @client_pong_ids_received = Set.new
+      @client_invalidated_ids_received = Set.new
+      MessageDispatcher.configuration_server = self
+    end
+
+    # Redis system status information (an instance of class RedisServerInfo)
+    def redis
+      @redis ||= RedisServerInfo.new(config, :timeout => 3)
+    end
+
+    # Beetle::Client instance for communication with the RedisConfigurationServer
+    def beetle
+      @beetle ||= build_beetle
+    end
+
+    def config #:nodoc:
+      beetle.config
+    end
+
+    # start watching redis
+    def start
+      verify_redis_master_file_string
+      check_redis_configuration
+      redis.refresh
+      determine_initial_master
+      log_start
+      beetle.listen do
+        master_watcher.watch
+      end
+    end
+
+    # test if redis is currently being watched
+    def paused?
+      master_watcher.paused?
+    end
+
+    # called by the message dispatcher when a "pong" message from a RedisConfigurationClient is received
+    def pong(payload)
+      id = payload["id"]
+      token = payload["token"]
+      logger.info "Received pong message from id '#{id}' with token '#{token}'"
+      return unless redeem_token(token)
+      @client_pong_ids_received << id
+      if all_client_pong_ids_received?
+        logger.debug "All client pong messages received"
+        @available_timer.cancel if @available_timer
+        invalidate_current_master
+      end
+    end
+
+    # called by the message dispatcher when a "client_invalidated" message from a RedisConfigurationClient is received
+    def client_invalidated(payload)
+      id = payload["id"]
+      token = payload["token"]
+      logger.info "Received client_invalidated message from id '#{id}' with token '#{token}'"
+      return unless redeem_token(token)
+      @client_invalidated_ids_received << id
+      if all_client_invalidated_ids_received?
+        logger.debug "All client invalidated messages received"
+        @invalidate_timer.cancel if @invalidate_timer
+        switch_master
+      end
+    end
+
+    # called from RedisWatcher when watched redis becomes unavailable
+    def master_unavailable!
+      msg = "Redis master '#{current_master.server}' not available"
+      master_watcher.pause
+      logger.warn(msg)
+      beetle.publish(:system_notification, {"message" => msg}.to_json)
+
+      if @client_ids.empty?
+        switch_master
+      else
+        start_invalidation
+      end
+    end
+
+    # called from RedisWatcher when watched redis is available
+    def master_available!
+      publish_master(current_master)
+      configure_slaves(current_master)
+    end
+
+    # check whether the current master is still online and still a master
+    def master_available?
+      redis.masters.include?(current_master)
+    end
+
+    private
+
+    def check_redis_configuration
+      raise ConfigurationError.new("Redis failover needs two or more redis servers") if redis.instances.size < 2
+    end
+
+    def log_start
+      logger.info "RedisConfigurationServer starting"
+      logger.info "AMQP servers  : #{config.servers}"
+      logger.info "Client ids    : #{config.redis_configuration_client_ids}"
+      logger.info "Redis servers : #{config.redis_servers}"
+      logger.info "Redis master  : #{current_master.server}"
+    end
+
+    def build_beetle
+      system = Beetle.config.system_name
+      Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
+        config.message :client_invalidated
+        config.queue   :client_invalidated, :amqp_name => "#{system}_client_invalidated"
+        config.message :pong
+        config.queue   :pong, :amqp_name => "#{system}_pong"
+        config.message :ping
+        config.message :invalidate
+        config.message :reconfigure
+        config.message :system_notification
+
+        config.handler [:pong, :client_invalidated], MessageDispatcher
+      end
+    end
+
+    def master_watcher
+      @master_watcher ||= RedisWatcher.new(self)
+    end
+
+    def determine_initial_master
+      if master_file_exists? && @current_master = redis_master_from_master_file
+        if redis.slaves.include?(current_master)
+          master_unavailable!
+        elsif redis.unknowns.include?(current_master)
+          master_unavailable!
+        end
+      else
+        write_redis_master_file(current_master.server) if @current_master = redis.auto_detect_master
+      end
+      current_master or raise NoRedisMaster.new("failed to determine initial redis master")
+    end
+
+    def determine_new_master
+      redis.unknowns.include?(current_master) ? redis.slaves_of(current_master).first : current_master
+    end
+
+    def redeem_token(token)
+      valid_token = token == @current_token
+      logger.info "Ignored message (token was '#{token.inspect}', but expected '#{@current_token.inspect}')" unless valid_token
+      valid_token
+    end
+
+    def start_invalidation
+      @client_pong_ids_received.clear
+      @client_invalidated_ids_received.clear
+      check_all_clients_available
+    end
+
+    def check_all_clients_available
+      generate_new_token
+      beetle.publish(:ping, payload_with_current_token)
+      @available_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
+    end
+
+    def invalidate_current_master
+      generate_new_token
+      beetle.publish(:invalidate, payload_with_current_token)
+      @invalidate_timer = EM::Timer.new(config.redis_configuration_client_timeout) { cancel_invalidation }
+    end
+
+    def cancel_invalidation
+      logger.warn "Redis master invalidation cancelled: 'pong' received from '#{@client_pong_ids_received.to_a.join(',')}', 'client_invalidated' received from '#{@client_invalidated_ids_received.to_a.join(',')}'"
+      generate_new_token
+      master_watcher.continue
+    end
+
+    def generate_new_token
+      @current_token += 1
+    end
+
+    def payload_with_current_token(message = {})
+      message["token"] = @current_token
+      message.to_json
+    end
+
+    def all_client_pong_ids_received?
+      @client_ids == @client_pong_ids_received
+    end
+
+    def all_client_invalidated_ids_received?
+      @client_ids == @client_invalidated_ids_received
+    end
+
+    def switch_master
+      if new_master = determine_new_master
+        msg = "Setting redis master to '#{new_master.server}' (was '#{current_master.server}')"
+        logger.warn(msg)
+        beetle.publish(:system_notification, {"message" => msg}.to_json)
+
+        new_master.master!
+        @current_master = new_master
+      else
+        msg = "Redis master could not be switched, no slave available to become new master, promoting old master"
+        logger.error(msg)
+        beetle.publish(:system_notification, {"message" => msg}.to_json)
+      end
+
+      publish_master(current_master)
+      master_watcher.continue
+    end
+
+    def publish_master(master)
+      logger.info "Publishing reconfigure message with server '#{master.server}'"
+      beetle.publish(:reconfigure, payload_with_current_token({"server" => master.server}))
+    end
+
+    def configure_slaves(master)
+      (redis.masters-[master]).each do |r|
+        logger.info "Reconfiguring '#{r.server}' as a slave of '#{master.server}'"
+        r.slave_of!(master.host, master.port)
+      end
+    end
+
+    # Periodically checks a redis server for availability
+    class RedisWatcher #:nodoc:
+      include Logging
+
+      def initialize(configuration_server)
+        @configuration_server = configuration_server
+        @retries = 0
+        @paused = true
+        @master_retry_interval = configuration_server.config.redis_configuration_master_retry_interval
+        @master_retries = configuration_server.config.redis_configuration_master_retries
+      end
+
+      def pause
+        logger.info "Pause checking availability of redis servers"
+        @watch_timer.cancel if @watch_timer
+        @watch_timer = nil
+        @paused = true
+      end
+
+      def watch
+        @watch_timer ||=
+          begin
+            logger.info "Start watching redis servers every #{@master_retry_interval} seconds"
+            EventMachine::add_periodic_timer(@master_retry_interval) { check_availability }
+          end
+        @paused = false
+      end
+      alias continue watch
+
+      def paused?
+        @paused
+      end
+
+      private
+
+      def check_availability
+        @configuration_server.redis.refresh
+        if @configuration_server.master_available?
+          @configuration_server.master_available!
+        else
+          logger.warn "Redis master not available! (Retries left: #{@master_retries - (@retries + 1)})"
+          if (@retries+=1) >= @master_retries
+            @retries = 0
+            @configuration_server.master_unavailable!
+          end
+        end
+      end
+    end
+
+    # Dispatches messages from the queue to methods in RedisConfigurationServer
+    class MessageDispatcher < Beetle::Handler #:nodoc:
+      cattr_accessor :configuration_server
+      def process
+        @@configuration_server.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
+      end
+    end
+  end
+end

data/lib/beetle/redis_ext.rb

@@ -0,0 +1,79 @@
+# Redis convenience and compatibility layer
+class Redis #:nodoc:
+  def self.from_server_string(server_string, options = {})
+    host, port = server_string.split(':')
+    options = {:host => host, :port => port}.update(options)
+    new(options)
+  end
+
+  # Redis 2 removed some useful methods. add them back.
+  def host; @client.host; end
+  def port; @client.port; end
+  def server; "#{host}:#{port}"; end
+
+  def master!
+    slaveof("no", "one")
+  end
+
+  def slave_of!(host, port)
+    slaveof(host, port)
+  end
+
+  # Redis 2 tries to establish a connection on inspect. this is evil!
+  def inspect
+    super
+  end
+
+  def info_with_rescue
+    info
+  rescue Exception
+    {}
+  end
+
+  def available?
+    info_with_rescue != {}
+  end
+
+  def role
+    info_with_rescue["role"] || "unknown"
+  end
+
+  def master?
+    role == "master"
+  end
+
+  def slave?
+    role == "slave"
+  end
+
+  def slave_of?(host, port)
+    info = info_with_rescue
+    info["role"] == "slave" && info["master_host"] == host && info["master_port"] == port.to_s
+  end
+end
+
+class Redis::Client #:nodoc:
+  protected
+  def connect_to(host, port)
+    if @timeout != 0 and Redis::Timer
+      begin
+        Redis::Timer.timeout(@timeout){ @sock = TCPSocket.new(host, port) }
+      rescue Timeout::Error
+        @sock = nil
+        raise Timeout::Error, "Timeout connecting to the server"
+      end
+    else
+      @sock = TCPSocket.new(host, port)
+    end
+
+    @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+    # If the timeout is set we set the low level socket options in order
+    # to make sure a blocking read will return after the specified number
+    # of seconds. This hack is from memcached ruby client.
+    self.timeout = @timeout
+
+  rescue Errno::ECONNREFUSED
+    raise Errno::ECONNREFUSED, "Unable to connect to Redis on #{host}:#{port}"
+  end
+end

data/lib/beetle/redis_master_file.rb

@@ -0,0 +1,35 @@
+module Beetle
+  module RedisMasterFile #:nodoc:
+    private
+    def master_file_exists?
+      File.exist?(master_file)
+    end
+
+    def redis_master_from_master_file
+      redis.find(read_redis_master_file)
+    end
+
+    def clear_redis_master_file
+      write_redis_master_file("")
+    end
+
+    def read_redis_master_file
+      File.read(master_file).chomp
+    end
+
+    def write_redis_master_file(redis_server_string)
+      logger.warn "Writing '#{redis_server_string}' to redis master file '#{master_file}'"
+      File.open(master_file, "w"){|f| f.puts redis_server_string }
+    end
+
+    def master_file
+      config.redis_server
+    end
+
+    def verify_redis_master_file_string
+      if master_file =~ /^[0-9a-z.]+:[0-9]+$/
+        raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
+      end
+    end
+  end
+end

data/lib/beetle/redis_server_info.rb

@@ -0,0 +1,65 @@
+module Beetle
+  # class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
+  class RedisServerInfo
+    include Logging
+
+    def initialize(config, options) #:nodoc:
+      @config = config
+      @options = options
+      reset
+    end
+
+    # all configured redis servers
+    def instances
+      @instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
+    end
+
+    # fetches the server from the insatnces whith the given <tt>server</tt> string
+    def find(server)
+      instances.find{|r| r.server == server}
+    end
+
+    # refresh connectivity/role information
+    def refresh
+      logger.debug "Updating redis server info"
+      reset
+      instances.each {|r| @server_info[r.role] << r}
+    end
+
+    # subset of instances which are masters
+    def masters
+      @server_info["master"]
+    end
+
+    # subset of instances which are slaves
+    def slaves
+      @server_info["slave"]
+    end
+
+    # subset of instances which are not reachable
+    def unknowns
+      @server_info["unknown"]
+    end
+
+    # subset of instances which are set up as slaves of the given <tt>master</tt>
+    def slaves_of(master)
+      slaves.select{|r| r.slave_of?(master.host, master.port)}
+    end
+
+    # determine a master if we have one master and all other servers are slaves
+    def auto_detect_master
+      return nil unless master_and_slaves_reachable?
+      masters.first
+    end
+
+    private
+
+    def master_and_slaves_reachable?
+      masters.size == 1 && slaves.size == instances.size - 1
+    end
+
+    def reset
+      @server_info = Hash.new {|h,k| h[k]= []}
+    end
+  end
+end

data/lib/beetle/subscriber.rb

@@ -10,7 +10,7 @@ module Beetle
       @mqs = {}
     end
 
-    # the client calls this method to subcsribe to all queues on all servers which have
+    # the client calls this method to subscribe to all queues on all servers which have
     # handlers registered for the given list of messages. this method does the following
     # things:
     #
@@ -105,6 +105,9 @@ module Beetle
             sleep 1
             mq(server).recover
           elsif reply_to = header.properties[:reply_to]
+            # require 'ruby-debug'
+            # Debugger.start
+            # debugger
             status = result == Beetle::RC::OK ? "OK" : "FAILED"
             exchange = MQ::Exchange.new(mq(server), :direct, "", :key => reply_to)
             exchange.publish(m.handler_result.to_s, :headers => {:status => status})

data/lib/beetle.rb

@@ -17,8 +17,6 @@ module Beetle
   class UnknownQueue < Error; end
   # raised when no redis master server can be found
   class NoRedisMaster < Error; end
-  # raised when two redis master servers are found
-  class TwoRedisMasters < Error; end
 
   # AMQP options for exchange creation
   EXCHANGE_CREATION_KEYS  = [:auto_delete, :durable, :internal, :nowait, :passive]
@@ -37,6 +35,8 @@ module Beetle
     autoload File.basename(libfile)[/^(.*)\.rb$/, 1].classify, libfile
   end
 
+  require "#{lib_dir}/redis_ext"
+
   # returns the default configuration object and yields it if a block is given
   def self.config
     #:yields: config

data/test/beetle/configuration_test.rb

@@ -1,5 +1,17 @@
 require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
-
+require 'tempfile'
 
 module Beetle
-end
+  class ConfigurationTest < Test::Unit::TestCase
+    test "should load it's settings from a config file if that file exists" do
+      config    = Configuration.new
+      old_value = config.gc_threshold
+      new_value = old_value + 1
+      config_file_content = "gc_threshold: #{new_value}\n"
+      IO.expects(:read).returns(config_file_content)
+
+      config.config_file = "some/path/to/a/file"
+      assert_equal new_value, config.gc_threshold
+    end
+  end
+end

data/test/beetle/deduplication_store_test.rb

@@ -14,75 +14,93 @@ module Beetle
     end
 
     test "msetnx returns 0 or 1" do
-      assert_equal 1, @r.msetnx("a" => 1, "b" => 2)
+      assert_equal 1, @r.msetnx("a", 1, "b", 2)
       assert_equal "1", @r.get("a")
       assert_equal "2", @r.get("b")
-      assert_equal 0, @r.msetnx("a" => 3, "b" => 4)
+      assert_equal 0, @r.msetnx("a", 3, "b", 4)
       assert_equal "1", @r.get("a")
       assert_equal "2", @r.get("b")
     end
   end
 
-  class RedisFailoverTest < Test::Unit::TestCase
+  class RedisServerStringTest < Test::Unit::TestCase
+    def setup
+      @original_redis_server = Beetle.config.redis_server
+      @store = DeduplicationStore.new
+      @server_string = "my_test_host_from_file:9999"
+      Beetle.config.redis_server = @server_string
+    end
+
+    def teardown
+      Beetle.config.redis_server = @original_redis_server
+    end
+
+    test "redis should match the redis server string" do
+      assert_equal @server_string, @store.redis.server
+    end
+  end
+
+  class RedisServerFileTest < Test::Unit::TestCase
     def setup
-      @store = DeduplicationStore.new("localhost:1, localhost:2")
+      @original_redis_server = Beetle.config.redis_server
+      @store = DeduplicationStore.new
+      @server_string = "my_test_host_from_file:6379"
+      Beetle.config.redis_server = redis_test_master_file(@server_string)
+    end
+
+    def teardown
+      Beetle.config.redis_server = @original_redis_server
+    end
+
+    test "redis should match the redis master file" do
+      assert_equal @server_string, @store.redis.server
     end
 
-    test "redis instances should be created for all servers" do
-      instances = @store.redis_instances
-      assert_equal ["localhost:1", "localhost:2" ], instances.map(&:server)
+    test "redis should be nil if the redis master file is blank" do
+      redis_test_master_file("")
+      assert_nil @store.redis
     end
 
-    test "searching a redis master should find one if there is one" do
-      instances = @store.redis_instances
-      instances.first.expects(:info).returns(:role => "slave")
-      instances.second.expects(:info).returns(:role => "master")
-      assert_equal instances.second, @store.redis
+    test "should keep using the current redis if the redis master file hasn't changed since the last request" do
+      @store.expects(:read_master_file).once.returns("localhost:1")
+      2.times { @store.redis }
     end
 
-    test "searching a redis master should find one even if one cannot be accessed" do
-      instances = @store.redis_instances
-      instances.first.expects(:info).raises("murks")
-      instances.second.expects(:info).returns(:role => "master")
-      assert_equal instances.second, @store.redis
+    test "should return nil if the master file doesn't exist" do
+      Beetle.config.redis_server = "/tmp/__i_don_not_exist__.txt"
+      assert_equal nil, @store.redis_master_from_master_file
     end
 
-    test "searching a redis master should raise an exception if there is none" do
-      instances = @store.redis_instances
-      instances.first.expects(:info).returns(:role => "slave")
-      instances.second.expects(:info).returns(:role => "slave")
-      assert_raises(NoRedisMaster) { @store.find_redis_master }
+    private
+    def redis_test_master_file(server_string)
+      path = File.expand_path("../../../tmp/redis-master-for-unit-tests", __FILE__)
+      File.open(path, "w"){|f| f.puts server_string}
+      path
     end
+  end
 
-    test "searching a redis master should raise an exception if there is more than one" do
-      instances = @store.redis_instances
-      instances.first.expects(:info).returns(:role => "master")
-      instances.second.expects(:info).returns(:role => "master")
-      assert_raises(TwoRedisMasters) { @store.find_redis_master }
+  class RedisFailoverTest < Test::Unit::TestCase
+    def setup
+      @store = DeduplicationStore.new
+      Beetle.config.expects(:redis_failover_timeout).returns(1)
     end
 
     test "a redis operation protected with a redis failover block should succeed if it can find a new master" do
-      instances = @store.redis_instances
+      redis1 = stub("redis 1")
+      redis2 = stub("redis 2")
       s = sequence("redis accesses")
-      instances.first.expects(:info).returns(:role => "master").in_sequence(s)
-      instances.second.expects(:info).returns(:role => "slave").in_sequence(s)
-      assert_equal instances.first, @store.redis
-      instances.first.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
-      instances.first.expects(:info).raises("disconnected").in_sequence(s)
-      instances.second.expects(:info).returns(:role => "master").in_sequence(s)
-      instances.second.expects(:get).with("foo:x").returns("42").in_sequence(s)
+      @store.expects(:redis).returns(redis1).in_sequence(s)
+      redis1.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
+      @store.expects(:redis).returns(redis2).in_sequence(s)
+      redis2.expects(:get).with("foo:x").returns("42").in_sequence(s)
       assert_equal("42", @store.get("foo", "x"))
     end
 
     test "a redis operation protected with a redis failover block should fail if it cannot find a new master" do
-      instances = @store.redis_instances
-      instances.first.expects(:info).returns(:role => "master")
-      instances.second.expects(:info).returns(:role => "slave")
-      assert_equal instances.first, @store.redis
-      instances.first.stubs(:get).with("foo:x").raises("disconnected")
-      instances.first.stubs(:info).raises("disconnected")
-      instances.second.stubs(:info).returns(:role => "slave")
-      @store.expects(:sleep).times(119)
+      redis1 = stub()
+      @store.stubs(:redis).returns(redis1)
+      redis1.stubs(:get).with("foo:x").raises("disconnected")
+      @store.stubs(:sleep)
       assert_raises(NoRedisMaster) { @store.get("foo", "x") }
     end
   end