spbtv_redis_failover 1.0.2.1

data/lib/redis_failover/node_snapshot.rb

@@ -0,0 +1,81 @@
+module RedisFailover
+  # Represents a snapshot of a particular redis node as seen by all currently running
+  # redis node managers.
+  class NodeSnapshot
+    # @return [String] the redis node
+    attr_reader :node
+
+    # Creates a new instance.
+    #
+    # @param [String] the redis node
+    # @see NodeManager#initialize
+    def initialize(node)
+      @node = node
+      @available = {}
+      @unavailable = []
+    end
+
+    # Declares this node available by the specified node manager.
+    #
+    # @param [String] node_manager the node manager id
+    # @param [Integer] latency the latency
+    def viewable_by(node_manager, latency)
+      @available[node_manager] = latency
+    end
+
+    # Determines if this node is viewable by a node manager.
+    #
+    # @param [String] node_manager the node manager id
+    def viewable_by?(node_manager)
+      @available.key?(node_manager)
+    end
+
+    # Declares this node unavailable by the specified node manager.
+    #
+    # @param [String] node_manager the node manager id
+    def unviewable_by(node_manager)
+      @unavailable << node_manager
+    end
+
+    # @return [Integer] the number of node managers saying
+    # this node is available
+    def available_count
+      @available.size
+    end
+
+    # @return [Integer] the number of node managers saying
+    # this node is unavailable
+    def unavailable_count
+      @unavailable.size
+    end
+
+    # @return [Integer] the average available latency
+    def avg_latency
+      return if @available.empty?
+      @available.values.inject(0) { |sum, n| sum + n } / @available.size
+    end
+
+    # @return [Array<String>] all node managers involved in this snapshot
+    def node_managers
+      (@available.keys + @unavailable).uniq
+    end
+
+    # @return [Boolean] true if all node managers indicated that this
+    # node was viewable
+    def all_available?
+      available_count > 0 && unavailable_count == 0
+    end
+
+    # @return [Boolean] true if all node managers indicated that this
+    # node was unviewable
+    def all_unavailable?
+      unavailable_count > 0 && available_count == 0
+    end
+
+    # @return [String] a friendly representation of this node snapshot
+    def to_s
+      'Node %s available by %p, unavailable by %p (%d up, %d down)' %
+        [node, @available, @unavailable, available_count, unavailable_count]
+    end
+  end
+end

data/lib/redis_failover/node_strategy.rb

@@ -0,0 +1,34 @@
+module RedisFailover
+  # Base class for strategies that determine node availability.
+  class NodeStrategy
+    include Util
+
+    # Loads a strategy based on the given name.
+    #
+    # @param [String, Symbol] name the strategy name
+    # @return [Object] a new strategy instance
+    def self.for(name)
+      require "redis_failover/node_strategy/#{name.downcase}"
+      const_get(name.capitalize).new
+    rescue LoadError, NameError
+      raise "Failed to find node strategy: #{name}"
+    end
+
+    # Returns the state determined by this strategy.
+    #
+    # @param [Node] the node to handle
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    # @return [Symbol] the status
+    def determine_state(node, snapshots)
+      raise NotImplementedError
+    end
+
+    # Logs a node as being unavailable.
+    #
+    # @param [Node] node the node
+    # @param [NodeSnapshot] snapshot the node snapshot
+    def log_unavailable(node, snapshot)
+      logger.info("#{self.class} marking #{node} as unavailable. Snapshot: #{snapshot}")
+    end
+  end
+end

data/lib/redis_failover/node_strategy/consensus.rb

@@ -0,0 +1,18 @@
+module RedisFailover
+  class NodeStrategy
+    # Consensus strategy only marks the node as unavailable if all members of the
+    # snapshot indicate that the node is down.
+    class Consensus < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.all_unavailable?
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_strategy/majority.rb

@@ -0,0 +1,18 @@
+module RedisFailover
+  class NodeStrategy
+    # Majority strategy only marks the node as unavailable if a majority of the
+    # snapshot indicates that the node is down.
+    class Majority < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.unavailable_count > snapshot.available_count
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_strategy/single.rb

@@ -0,0 +1,17 @@
+module RedisFailover
+  class NodeStrategy
+    # Marks the node as unavailable if any node manager reports the node as down.
+    class Single < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.unavailable_count > 0
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_watcher.rb

@@ -0,0 +1,83 @@
+module RedisFailover
+  # NodeWatcher periodically monitors a specific redis node for its availability.
+  # NodeWatcher instances periodically report a redis node's current state
+  # to the NodeManager for proper handling.
+  class NodeWatcher
+    include Util
+
+    # Time to sleep before checking on the monitored node's status.
+    WATCHER_SLEEP_TIME = 2
+
+    # Creates a new instance.
+    #
+    # @param [NodeManager] manager the node manager
+    # @param [Node] node the node to watch
+    # @param [Integer] max_failures the max failues before reporting node as down
+    def initialize(manager, node, max_failures)
+      @manager = manager
+      @node = node
+      @max_failures = max_failures
+      @monitor_thread = nil
+      @done = false
+    end
+
+    # Starts the node watcher.
+    #
+    # @note this method returns immediately and causes monitoring to be
+    #   performed in a new background thread
+    def watch
+      @monitor_thread ||= Thread.new { monitor_node }
+      self
+    end
+
+    # Performs a graceful shutdown of this watcher.
+    def shutdown
+      @done = true
+      begin
+        @node.wakeup
+      rescue
+        # best effort
+      end
+      @monitor_thread.join
+    rescue => ex
+      logger.warn("Failed to gracefully shutdown watcher for #{@node}")
+    end
+
+    private
+
+    # Periodically monitors the redis node and reports state changes to
+    # the {RedisFailover::NodeManager}.
+    def monitor_node
+      failures = 0
+
+      loop do
+        begin
+          break if @done
+          sleep(WATCHER_SLEEP_TIME)
+          latency = Benchmark.realtime { @node.ping }
+          failures = 0
+          notify(:available, latency)
+          @node.wait
+        rescue NodeUnavailableError => ex
+          logger.debug("Failed to communicate with node #{@node}: #{ex.inspect}")
+          failures += 1
+          if failures >= @max_failures
+            notify(:unavailable)
+            failures = 0
+          end
+        rescue Exception => ex
+          logger.error("Unexpected error while monitoring node #{@node}: #{ex.inspect}")
+          logger.error(ex.backtrace.join("\n"))
+        end
+      end
+    end
+
+    # Notifies the manager of a node's state.
+    #
+    # @param [Symbol] state the node's state
+    # @param [Integer] latency an optional latency
+    def notify(state, latency = nil)
+      @manager.notify_state(@node, state, latency)
+    end
+  end
+end

data/lib/redis_failover/runner.rb

@@ -0,0 +1,27 @@
+module RedisFailover
+  # Runner is responsible for bootstrapping the Node Manager.
+  class Runner
+    # Launches the Node Manager in a background thread.
+    #
+    # @param [Array] options the command-line options
+    # @note this method blocks and does not return until the
+    #   Node Manager is gracefully stopped
+    def self.run(options)
+      options = CLI.parse(options)
+      node_manager = NodeManager.new(options)
+      trap_signals(node_manager)
+      node_manager.start
+    end
+
+    # Traps shutdown signals.
+    # @param [NodeManager] node_manager the node manager
+    def self.trap_signals(node_manager)
+      [:INT, :TERM].each do |signal|
+        trap(signal) do
+          node_manager.shutdown
+        end
+      end
+    end
+    private_class_method :trap_signals
+  end
+end

data/lib/redis_failover/util.rb

@@ -0,0 +1,137 @@
+require 'redis_failover/errors'
+
+module RedisFailover
+  # Common utiilty methods and constants.
+  module Util
+    extend self
+
+    # Redis read operations that are automatically dispatched to slaves. Any
+    # operation not listed here will be dispatched to the master.
+    REDIS_READ_OPS = Set[
+      :echo,
+      :exists,
+      :get,
+      :getbit,
+      :getrange,
+      :hexists,
+      :hget,
+      :hgetall,
+      :hkeys,
+      :hlen,
+      :hmget,
+      :hvals,
+      :keys,
+      :lindex,
+      :llen,
+      :lrange,
+      :mapped_hmget,
+      :mapped_mget,
+      :mget,
+      :scard,
+      :sdiff,
+      :sinter,
+      :sismember,
+      :smembers,
+      :srandmember,
+      :strlen,
+      :sunion,
+      :type,
+      :zcard,
+      :zcount,
+      :zrange,
+      :zrangebyscore,
+      :zrank,
+      :zrevrange,
+      :zrevrangebyscore,
+      :zrevrank,
+      :zscore
+    ].freeze
+
+    # Unsupported Redis operations. These don't make sense in a client
+    # that abstracts the master/slave servers.
+    UNSUPPORTED_OPS = Set[:select, :dbsize].freeze
+
+    # Default root node in ZK used for redis_failover.
+    DEFAULT_ROOT_ZNODE_PATH = '/redis_failover'.freeze
+
+    # Connectivity errors that the redis (<3.x) client raises.
+    REDIS_ERRORS = Errno.constants.map { |c| Errno.const_get(c) }
+
+    # Connectivity errors that the redis (>3.x) client raises.
+    REDIS_ERRORS << Redis::BaseError if Redis.const_defined?('BaseError')
+    REDIS_ERRORS.freeze
+
+    # ZK Errors
+    ZK_ERRORS = [
+      ZK::Exceptions::LockAssertionFailedError,
+      ZK::Exceptions::InterruptedSession,
+      ZK::Exceptions::Retryable,
+      Zookeeper::Exceptions::ContinuationTimeoutError
+    ].freeze
+
+    # Full set of errors related to connectivity.
+    CONNECTIVITY_ERRORS = [
+      RedisFailover::Error,
+      REDIS_ERRORS,
+      ZK_ERRORS
+    ].flatten.freeze
+
+    # Symbolizes the keys of the specified hash.
+    #
+    # @param [Hash] hash a hash for which keys should be symbolized
+    # @return [Hash] a new hash with symbolized keys
+    def symbolize_keys(hash)
+      Hash[hash.map { |k, v| [k.to_sym, v] }]
+    end
+
+    # Determines if two arrays are different.
+    #
+    # @param [Array] ary_a the first array
+    # @param [Array] ary_b the second array
+    # @return [Boolean] true if arrays are different, false otherwise
+    def different?(ary_a, ary_b)
+      ((ary_a | ary_b) - (ary_a & ary_b)).size > 0
+    end
+
+    # @return [Logger] the logger instance to use
+    def self.logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime.utc} RedisFailover #{Process.pid} #{severity}: #{msg}\n"
+        end
+        logger
+      end
+    end
+
+    # Sets a new logger to use.
+    #
+    # @param [Logger] logger a new logger to use
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    # @return [Logger] the logger instance to use
+    def logger
+      Util.logger
+    end
+
+    # Encodes the specified data in JSON format.
+    #
+    # @param [Object] data the data to encode
+    # @return [String] the JSON-encoded data
+    def encode(data)
+      MultiJson.encode(data)
+    end
+
+    # Decodes the specified JSON data.
+    #
+    # @param [String] data the JSON data to decode
+    # @return [Object] the decoded data
+    def decode(data)
+      return unless data
+      MultiJson.decode(data)
+    end
+  end
+end

data/lib/redis_failover/version.rb

@@ -0,0 +1,3 @@
+module RedisFailover
+  VERSION = '1.0.2.1'
+end

data/spbtv_redis_failover.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/redis_failover/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Ryan LeCompte"]
+  gem.email         = ["lecompte@gmail.com"]
+  gem.description   = %(redis_failover is a ZooKeeper-based automatic master/slave failover solution for Ruby)
+  gem.summary       = %(redis_failover is a ZooKeeper-based automatic master/slave failover solution for Ruby)
+  gem.homepage      = "http://github.com/ryanlecompte/redis_failover"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "spbtv_redis_failover"
+  gem.require_paths = ["lib"]
+  gem.version       = RedisFailover::VERSION
+
+  gem.add_dependency('redis', ['>= 2.2', '< 4'])
+  gem.add_dependency('redis-namespace')
+  gem.add_dependency('multi_json', '~> 1')
+  gem.add_dependency('zk', ['>= 1.9', '< 1.10'])
+
+  gem.add_development_dependency('rake', '< 11.0')
+  gem.add_development_dependency('rspec', '< 3.0')
+  gem.add_development_dependency('yard')
+end