nogara-redis_failover 0.8.9

data/lib/redis_failover/node_watcher.rb

@@ -0,0 +1,79 @@
+module RedisFailover
+  # NodeWatcher periodically monitors a specific redis node for its availability.
+  # NodeWatcher instances periodically report a redis node's current state
+  # to the NodeManager for proper handling.
+  class NodeWatcher
+    include Util
+
+    # Time to sleep before checking on the monitored node's status.
+    WATCHER_SLEEP_TIME = 2
+
+    # Creates a new instance.
+    #
+    # @param [NodeManager] manager the node manager
+    # @param [Node] node the node to watch
+    # @param [Integer] max_failures the max failues before reporting node as down
+    def initialize(manager, node, max_failures)
+      @manager = manager
+      @node = node
+      @max_failures = max_failures
+      @monitor_thread = nil
+      @done = false
+    end
+
+    # Starts the node watcher.
+    #
+    # @note this method returns immediately and causes monitoring to be
+    #   performed in a new background thread
+    def watch
+      @monitor_thread ||= Thread.new { monitor_node }
+      self
+    end
+
+    # Performs a graceful shutdown of this watcher.
+    def shutdown
+      @done = true
+      @node.wakeup
+      @monitor_thread.join if @monitor_thread
+    rescue
+      # best effort
+    end
+
+    private
+
+    # Periodically monitors the redis node and reports state changes to
+    # the {RedisFailover::NodeManager}.
+    def monitor_node
+      failures = 0
+
+      loop do
+        begin
+          return if @done
+          sleep(WATCHER_SLEEP_TIME)
+          @node.ping
+          failures = 0
+
+          if @node.syncing_with_master?
+            notify(:syncing)
+          else
+            notify(:available)
+            @node.wait
+          end
+        rescue NodeUnavailableError
+          failures += 1
+          if failures >= @max_failures
+            notify(:unavailable)
+            failures = 0
+          end
+        end
+      end
+    end
+
+    # Notifies the manager of a node's state.
+    #
+    # @param [Symbol] state the node's state
+    def notify(state)
+      @manager.notify_state(@node, state)
+    end
+  end
+end

data/lib/redis_failover/runner.rb

@@ -0,0 +1,28 @@
+module RedisFailover
+  # Runner is responsible for bootstrapping the Node Manager.
+  class Runner
+    # Launches the Node Manager in a background thread.
+    #
+    # @param [Array] options the command-line options
+    # @note this method blocks and does not return until the
+    #   Node Manager is gracefully stopped
+    def self.run(options)
+      options = CLI.parse(options)
+      @node_manager = NodeManager.new(options)
+      trap_signals
+      node_manager_thread = Thread.new { @node_manager.start }
+      node_manager_thread.join
+    end
+
+    # Traps shutdown signals.
+    def self.trap_signals
+      [:INT, :TERM].each do |signal|
+        trap(signal) do
+          Util.logger.info('Shutting down ...')
+          @node_manager.shutdown
+          exit(0)
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/util.rb

@@ -0,0 +1,83 @@
+require 'redis_failover/errors'
+
+module RedisFailover
+  # Common utiilty methods and constants.
+  module Util
+    extend self
+
+    # Default node in ZK that contains the current list of available redis nodes.
+    DEFAULT_ZNODE_PATH = '/redis_failover_nodes'.freeze
+
+    # Connectivity errors that the redis (<3.x) client raises.
+    REDIS_ERRORS = Errno.constants.map { |c| Errno.const_get(c) }
+
+    # Connectivity errors that the redis (>3.x) client raises.
+    REDIS_ERRORS << Redis::BaseError if Redis.const_defined?("BaseError")
+    REDIS_ERRORS.freeze
+
+    # Full set of errors related to connectivity.
+    CONNECTIVITY_ERRORS = [
+      RedisFailover::Error,
+      ZK::Exceptions::InterruptedSession,
+      REDIS_ERRORS
+    ].flatten.freeze
+
+    # Symbolizes the keys of the specified hash.
+    #
+    # @param [Hash] hash a hash for which keys should be symbolized
+    # @return [Hash] a new hash with symbolized keys
+    def symbolize_keys(hash)
+      Hash[hash.map { |k, v| [k.to_sym, v] }]
+    end
+
+    # Determines if two arrays are different.
+    #
+    # @param [Array] ary_a the first array
+    # @param [Array] ary_b the second array
+    # @return [Boolean] true if arrays are different, false otherwise
+    def different?(ary_a, ary_b)
+      ((ary_a | ary_b) - (ary_a & ary_b)).size > 0
+    end
+
+    # @return [Logger] the logger instance to use
+    def self.logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime.utc} RedisFailover #{Process.pid} #{severity}: #{msg}\n"
+        end
+        logger
+      end
+    end
+
+    # Sets a new logger to use.
+    #
+    # @param [Logger] logger a new logger to use
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    # @return [Logger] the logger instance to use
+    def logger
+      Util.logger
+    end
+
+    # Encodes the specified data in JSON format.
+    #
+    # @param [Object] data the data to encode
+    # @return [String] the JSON-encoded data
+    def encode(data)
+      MultiJson.encode(data)
+    end
+
+    # Decodes the specified JSON data.
+    #
+    # @param [String] data the JSON data to decode
+    # @return [Object] the decoded data
+    def decode(data)
+      return unless data
+      MultiJson.decode(data)
+    end
+  end
+end

data/lib/redis_failover/version.rb

@@ -0,0 +1,3 @@
+module RedisFailover
+  VERSION = '0.8.9'
+end

data/redis_failover.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/redis_failover/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Ryan LeCompte"]
+  gem.email         = ["lecompte@gmail.com"]
+  gem.description   = %(Redis Failover is a ZooKeeper-based automatic master/slave failover solution for Ruby)
+  gem.summary       = %(Redis Failover is a ZooKeeper-based automatic master/slave failover solution for Ruby)
+  gem.homepage      = "http://github.com/ryanlecompte/redis_failover"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "nogara-redis_failover"
+  gem.require_paths = ["lib"]
+  gem.version       = RedisFailover::VERSION
+
+  gem.add_dependency('redis', '~> 3')
+  gem.add_dependency('redis-namespace')
+  gem.add_dependency('multi_json', '~> 1')
+  gem.add_dependency('zk', '~> 1.6')
+
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('rspec')
+  gem.add_development_dependency('yard')
+end

data/spec/cli_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe CLI do
+    describe '.parse' do
+      it 'properly parses redis nodes' do
+        opts = CLI.parse(['-n host1:1,host2:2,host3:3', '-z localhost:1111'])
+        opts[:nodes].should == [
+          {:host => 'host1', :port => '1'},
+          {:host => 'host2', :port => '2'},
+          {:host => 'host3', :port => '3'}
+        ]
+      end
+
+      it 'properly parses ZooKeeper servers' do
+        opts = CLI.parse(['-n host1:1,host2:2,host3:3', '-z localhost:1111'])
+        opts[:zkservers].should == 'localhost:1111'
+      end
+
+      it 'properly parses a redis password' do
+        opts = CLI.parse(['-n host:port', '-z localhost:1111', '-p redis_pass'])
+        opts[:nodes].should == [{
+          :host => 'host',
+          :port => 'port',
+          :password => 'redis_pass'
+        }]
+      end
+
+      it 'properly parses max node failures' do
+        opts = CLI.parse([
+          '-n host:port',
+          '-z localhost:1111',
+          '-p redis_pass',
+          '--max-failures',
+          '1'])
+        opts[:max_failures].should == 1
+      end
+
+      it 'properly parses a chroot' do
+        opts = CLI.parse(['-n host:port', '-z localhost:1111', '--with-chroot', '/with/chroot/from/command/line'])
+        opts[:nodes].should == [{
+          :host => 'host',
+          :port => 'port',
+        }]
+
+        opts[:chroot].should == '/with/chroot/from/command/line'
+      end
+
+      it 'properly parses the config file' do
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/single_environment.yml"])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments.yml", '-E', 'development'])
+        opts[:zkservers].should == 'localhost:2181'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments.yml", '-E', 'staging'])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+      end
+
+      it 'properly parses the config file that include chroot' do
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/single_environment_with_chroot.yml"])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+        opts[:chroot].should == '/with/chroot'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments_with_chroot.yml", '-E', 'development'])
+        opts[:zkservers].should == 'localhost:2181'
+        opts[:chroot].should == '/with/chroot_development'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments_with_chroot.yml", '-E', 'staging'])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+        opts[:chroot].should == '/with/chroot_staging'
+      end
+    end
+  end
+end

data/spec/client_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+module RedisFailover
+  Client::Redis = RedisStub
+  class ClientStub < Client
+    def current_master
+      @master
+    end
+
+    def current_slaves
+      @slaves
+    end
+
+    def fetch_nodes
+      {
+        :master => 'localhost:6379',
+        :slaves => ['localhost:1111'],
+        :unavailable => []
+      }
+    end
+
+    def setup_zk
+      @zk = NullObject.new
+      update_znode_timestamp
+    end
+  end
+
+  describe Client do
+    let(:client) { ClientStub.new(:zkservers => 'localhost:9281') }
+
+    describe '#build_clients' do
+      it 'properly parses master' do
+        client.current_master.to_s.should == 'localhost:6379'
+      end
+
+      it 'properly parses slaves' do
+        client.current_slaves.first.to_s.should == 'localhost:1111'
+      end
+    end
+
+    describe '#dispatch' do
+      it 'routes write operations to master' do
+        called = false
+        client.current_master.define_singleton_method(:del) do |*args|
+          called = true
+        end
+        client.del('foo')
+        called.should be_true
+      end
+
+      it 'routes read operations to a slave' do
+        called = false
+        client.current_slaves.first.change_role_to('slave')
+        client.current_slaves.first.define_singleton_method(:get) do |*args|
+          called = true
+        end
+        client.get('foo')
+        called.should be_true
+      end
+
+      it 'reconnects when node is unavailable' do
+        class << client
+          attr_reader :reconnected
+          def build_clients
+            @reconnected = true
+            super
+          end
+
+          def fetch_nodes
+            @calls ||= 0
+            {
+              :master => "localhost:222#{@calls += 1}",
+              :slaves => ['localhost:1111'],
+              :unavailable => []
+            }
+          end
+        end
+
+        client.current_master.make_unavailable!
+        client.del('foo')
+        client.reconnected.should be_true
+      end
+
+      it 'properly detects when a node has changed roles' do
+        client.current_master.change_role_to('slave')
+        expect { client.send(:master) }.to raise_error(InvalidNodeRoleError)
+      end
+
+      it 'raises error for unsupported operations' do
+        expect { client.select }.to raise_error(UnsupportedOperationError)
+      end
+
+      it 'attempts ZK reconnect when no communication from Node Manager within certain time window' do
+        client.instance_variable_set(:@last_znode_timestamp, Time.at(0))
+        client.should_receive(:build_clients)
+        client.del('foo')
+      end
+    end
+  end
+end

data/spec/node_manager_spec.rb

@@ -0,0 +1,112 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeManager do
+    let(:manager) { NodeManagerStub.new({}) }
+
+    before(:each) do
+      manager.discover_nodes
+    end
+
+    describe '#nodes' do
+      it 'returns current master and slave nodes' do
+        manager.current_nodes.should == {
+          :master => 'master:6379',
+          :slaves => ['slave:6379'],
+          :unavailable => []
+        }
+      end
+    end
+
+    describe '#handle_unavailable' do
+      context 'slave dies' do
+        it 'moves slave to unavailable list' do
+          slave = manager.slaves.first
+          manager.force_unavailable(slave)
+          manager.current_nodes[:unavailable].should include(slave.to_s)
+        end
+      end
+
+      context 'master dies' do
+        before(:each) do
+          @slave = manager.slaves.first
+          @master = manager.master
+          manager.force_unavailable(@master)
+        end
+
+        it 'promotes slave to master' do
+          manager.master.should == @slave
+        end
+
+        it 'moves master to unavailable list' do
+          manager.current_nodes[:unavailable].should include(@master.to_s)
+        end
+      end
+    end
+
+    describe '#handle_available' do
+      before(:each) do
+        # force to be unavailable first
+        @slave = manager.slaves.first
+        manager.force_unavailable(@slave)
+      end
+
+      context 'slave node with a master present' do
+        it 'removes slave from unavailable list' do
+          manager.force_available(@slave)
+          manager.current_nodes[:unavailable].should be_empty
+          manager.current_nodes[:slaves].should include(@slave.to_s)
+        end
+
+        it 'makes node a slave of new master' do
+          manager.master = Node.new(:host => 'foo', :port => '7892')
+          manager.force_available(@slave)
+          @slave.fetch_info.should == {
+            :role => 'slave',
+            :master_host => 'foo',
+            :master_port => '7892'}
+        end
+
+        it 'does not invoke slaveof operation if master has not changed' do
+          @slave.redis.should_not_receive(:slaveof)
+          manager.force_available(@slave)
+        end
+      end
+
+      context 'slave node with no master present' do
+        before(:each) do
+          @master = manager.master
+          manager.force_unavailable(@master)
+        end
+
+        it 'promotes slave to master' do
+          manager.force_available(@slave)
+          manager.master.should == @slave
+        end
+
+        it 'slaves list remains empty' do
+          manager.current_nodes[:slaves].should be_empty
+        end
+      end
+    end
+
+    describe '#handle_syncing' do
+      context 'prohibits stale reads' do
+        it 'adds node to unavailable list' do
+          slave = manager.slaves.first
+          manager.force_syncing(slave, false)
+          manager.current_nodes[:unavailable].should include(slave.to_s)
+        end
+      end
+
+      context 'allows stale reads' do
+        it 'makes node available' do
+          slave = manager.slaves.first
+          manager.force_syncing(slave, true)
+          manager.current_nodes[:unavailable].should_not include(slave.to_s)
+          manager.current_nodes[:slaves].should include(slave.to_s)
+        end
+      end
+    end
+  end
+end