redis_failover 0.1.0

data/lib/redis_failover/node.rb

@@ -0,0 +1,101 @@
+module RedisFailover
+  # Represents a redis node (master or slave).
+  class Node
+    include Util
+
+    attr_reader :host, :port
+
+    def initialize(options = {})
+      @host = options.fetch(:host) { raise InvalidNodeError, 'missing host'}
+      @port = Integer(options[:port] || 6379)
+      @password = options[:password]
+    end
+
+    def ping
+      perform_operation do
+        redis.ping
+      end
+    end
+
+    def master?
+      role == 'master'
+    end
+
+    def slave?
+      !master?
+    end
+
+    def wait_until_unreachable
+      perform_operation do
+        redis.blpop(wait_key, 0)
+        redis.del(wait_key)
+      end
+    end
+
+    def stop_waiting
+      perform_operation do
+        redis.lpush(wait_key, '1')
+      end
+    end
+
+    def make_slave!(master)
+      perform_operation do
+        redis.slaveof(master.host, master.port)
+      end
+    end
+
+    def make_master!
+      perform_operation do
+        # yes, this is a real redis operation!
+        redis.slaveof('no', 'one')
+      end
+    end
+
+    def inspect
+      "<RedisFailover::Node #{to_s}>"
+    end
+
+    def to_s
+      "#{@host}:#{@port}"
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Node)
+      return true if self.equal?(other)
+      [host, port] == [other.host, other.port]
+    end
+    alias_method :eql?, :==
+
+    def hash
+      to_s.hash
+    end
+
+    private
+
+    def role
+      fetch_info[:role]
+    end
+
+    def fetch_info
+      perform_operation do
+        symbolize_keys(redis.info)
+      end
+    end
+
+    def wait_key
+      @wait_key ||= "_redis_failover_#{SecureRandom.hex(32)}"
+    end
+
+    def redis
+      Redis.new(:host => @host, :password => @password, :port => @port)
+    rescue
+      raise NodeUnreachableError.new(self)
+    end
+
+    def perform_operation
+      yield
+    rescue
+      raise NodeUnreachableError.new(self)
+    end
+  end
+end

data/lib/redis_failover/node_manager.rb

@@ -0,0 +1,151 @@
+module RedisFailover
+  # NodeManager manages a list of redis nodes.
+  class NodeManager
+    include Util
+
+    def initialize(options)
+      @options = options
+      @master, @slaves = parse_nodes
+      @unreachable = []
+      @queue = Queue.new
+      @lock = Mutex.new
+    end
+
+    def start
+      spawn_watchers
+      handle_state_changes
+    end
+
+    def notify_state_change(node, state)
+      @queue << [node, state]
+    end
+
+    def nodes
+      @lock.synchronize do
+        {
+          :master => @master ? @master.to_s : nil,
+          :slaves => @slaves.map(&:to_s),
+          :unreachable => @unreachable.map(&:to_s)
+        }
+      end
+    end
+
+    def shutdown
+      @watchers.each(&:shutdown)
+    end
+
+    private
+
+    def handle_state_changes
+      while state_change = @queue.pop
+        @lock.synchronize do
+          node, state = state_change
+          begin
+            case state
+            when :unreachable then handle_unreachable(node)
+            when :reachable then handle_reachable(node)
+            else raise InvalidNodeStateError.new(node, state)
+            end
+          rescue NodeUnreachableError
+            # node suddenly became unreachable, silently
+            # handle since the watcher will take care of
+            # keeping track of the node
+          end
+        end
+      end
+    end
+
+    def handle_unreachable(node)
+      # no-op if we already know about this node
+      return if @unreachable.include?(node)
+      logger.info("Handling unreachable node: #{node}")
+
+      # find a new master if this node was a master
+      if node == @master
+        logger.info("Demoting currently unreachable master #{node}.")
+        promote_new_master
+      else
+        @slaves.delete(node)
+      end
+
+      @unreachable << node
+    end
+
+    def handle_reachable(node)
+      # no-op if we already know about this node
+      return if @master == node || @slaves.include?(node)
+      logger.info("Handling reachable node: #{node}")
+
+      @unreachable.delete(node)
+      @slaves << node
+      if @master
+        # master already exists, make a slave
+        node.make_slave!(@master)
+      else
+        # no master exists, make this the new master
+        promote_new_master
+      end
+    end
+
+    def promote_new_master
+      @master = nil
+
+      if @slaves.empty?
+        logger.error('Failed to promote a new master since no slaves available.')
+        return
+      end
+
+      # make a slave the new master
+      node = @slaves.pop
+      node.make_master!
+      @master = node
+
+      # switch existing slaves to point to new master
+      redirect_slaves
+      logger.info("Successfully promoted #{node} to master.")
+    end
+
+    def parse_nodes
+      nodes = @options[:nodes].map { |opts| Node.new(opts) }
+      raise NoMasterError unless master = find_master(nodes)
+      slaves = nodes - [master]
+
+      logger.info("Managing master (#{master}) and slaves" +
+        " (#{slaves.map(&:to_s).join(', ')})")
+
+      [master, slaves]
+    end
+
+    def spawn_watchers
+      @watchers = [@master, *@slaves].map do |node|
+          NodeWatcher.new(self, node,  @options[:max_failures] || 3)
+      end
+      @watchers.each(&:watch)
+    end
+
+    def find_master(nodes)
+      # try to find the master - if the actual master is currently
+      # down, it will be handled by its watcher
+      nodes.find do |node|
+        begin
+          node.master?
+        rescue NodeUnreachableError
+          # will eventually be handled by watcher
+          false
+        end
+      end
+    end
+
+    def redirect_slaves
+      # redirect each slave to a new master - if an actual slave is
+      # currently down, it will be handled by its watcher
+      @slaves.each do |slave|
+        begin
+          slave.make_slave!(@master)
+        rescue NodeUnreachableError
+          # will eventually be handled by watcher
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_watcher.rb

@@ -0,0 +1,46 @@
+module RedisFailover
+  # Watches a specific redis node for its reachability.
+  class NodeWatcher
+    def initialize(manager, node, max_failures)
+      @manager = manager
+      @node = node
+      @max_failures = max_failures
+      @monitor_thread = nil
+      @done = false
+    end
+
+    def watch
+      @monitor_thread = Thread.new { monitor_node }
+      self
+    end
+
+    def shutdown
+      @done = true
+      @node.stop_waiting
+      @monitor_thread.join if @monitor_thread
+    rescue
+      # best effort
+    end
+
+    private
+
+    def monitor_node
+      failures = 0
+
+      begin
+        return if @done
+        @node.ping
+        failures = 0
+        @manager.notify_state_change(@node, :reachable)
+        @node.wait_until_unreachable
+      rescue NodeUnreachableError
+        failures += 1
+        if failures >= @max_failures
+          @manager.notify_state_change(@node, :unreachable)
+          failures = 0
+        end
+        sleep(3) && retry
+      end
+    end
+  end
+end

data/lib/redis_failover/runner.rb

@@ -0,0 +1,31 @@
+module RedisFailover
+  # Runner is responsible for bootstrapping the redis failover server.
+  class Runner
+    def self.run(options)
+      options = CLI.parse(options)
+      Util.logger.info("Redis Failover Server starting on port #{options[:port]}")
+      Server.set(:port, options[:port])
+      @node_manager = NodeManager.new(options)
+      server_thread = Thread.new { Server.run! { |server| trap_signals } }
+      node_manager_thread = Thread.new { @node_manager.start }
+      [server_thread, node_manager_thread].each(&:join)
+    end
+
+    def self.node_manager
+      @node_manager
+    end
+
+    def self.trap_signals
+      [:INT, :TERM].each do |signal|
+        previous_signal = trap(signal) do
+          Util.logger.info('Shutting down ...')
+          if previous_signal && previous_signal.respond_to?(:call)
+            previous_signal.call
+          end
+          @node_manager.shutdown
+          exit(0)
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/server.rb

@@ -0,0 +1,13 @@
+require 'sinatra'
+
+module RedisFailover
+  # Serves as an endpoint for discovering the current redis master and slaves.
+  class Server < Sinatra::Base
+    disable :logging
+
+    get '/redis_servers' do
+      content_type :json
+      MultiJson.encode(Runner.node_manager.nodes)
+    end
+  end
+end

data/lib/redis_failover/util.rb

@@ -0,0 +1,29 @@
+module RedisFailover
+  # Common utiilty methods.
+  module Util
+    extend self
+
+    def symbolize_keys(hash)
+      Hash[hash.map { |k, v| [k.to_sym, v] }]
+    end
+
+    def self.logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime.utc} RedisFailover #{Process.pid} #{severity}: #{msg}\n"
+        end
+        logger
+      end
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def logger
+      Util.logger
+    end
+  end
+end

data/lib/redis_failover/version.rb

@@ -0,0 +1,3 @@
+module RedisFailover
+  VERSION = "0.1.0"
+end

data/redis_failover.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/redis_failover/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Ryan LeCompte"]
+  gem.email         = ["lecompte@gmail.com"]
+  gem.description   = %(Redis Failover provides a full automatic master/slave failover solution for Ruby)
+  gem.summary       = %(Redis Failover provides a full automatic master/slave failover solution for Ruby)
+  gem.homepage      = "http://github.com/ryanlecompte/redis_failover"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "redis_failover"
+  gem.require_paths = ["lib"]
+  gem.version       = RedisFailover::VERSION
+
+  gem.add_dependency('redis')
+  gem.add_dependency('redis-namespace')
+  gem.add_dependency('multi_json')
+  gem.add_dependency('sinatra')
+
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('rspec')
+end

data/spec/cli_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe CLI do
+    describe '.parse' do
+      it 'returns empty result for empty options' do
+        CLI.parse({}).should == {}
+      end
+
+      it 'properly parses a server port' do
+        opts = CLI.parse(['-P 2222'])
+        opts.should == {:port => 2222}
+      end
+
+      it 'properly parses redis nodes' do
+        opts = CLI.parse(['-n host1:1,host2:2,host3:3'])
+        opts[:nodes].should == [
+          {:host => 'host1', :port => '1'},
+          {:host => 'host2', :port => '2'},
+          {:host => 'host3', :port => '3'}
+        ]
+      end
+
+      it 'properly parses a redis password' do
+        opts = CLI.parse(['-n host:port', '-p redis_pass'])
+        opts[:nodes].should == [{
+          :host => 'host',
+          :port => 'port',
+          :password => 'redis_pass'
+        }]
+      end
+
+      it 'properly parses max node failures' do
+        opts = CLI.parse(['-n host:port', '-p redis_pass', '--max-failures', '1'])
+        opts[:max_failures].should == 1
+      end
+    end
+  end
+end