nogara-redis_failover 0.8.9 → 0.8.10

data/Changes.md

@@ -1,3 +1,13 @@
+HEAD
+-----------
+- Make Node Manager's lock path vary with its main znode. (Bira)
+- Node Manager's znode for holding current list of redis nodes is no longer ephemeral. This is unnecessary since the current master should only be changed by redis_failover.
+- Introduce :master_only option for RedisFailover::Client (disabled by default). This option configures the client to direct all read/write operations to the master.
+- Introduce :safe_mode option (enabled by default). This option configures the client to purge its redis clients when a ZK session expires or when the client hasn't recently heard from the node manager.
+- Introduce RedisFailover::Client#on_node_change callback notification for when the currently known list of master/slave redis nodes changes.
+- Added #current_master and #current_slaves to RedisFailover::Client. This is useful for programmatically doing things based on the current master/slaves.
+- redis_node_manager should start if no redis servers are available (#29)
+
 0.8.9
 -----------
 - Handle errors raised by redis 3.x client (tsilen) 

data/README.md

@@ -129,6 +129,16 @@ The full set of options that can be passed to RedisFailover::Client are:
      :logger        - logger override (optional)
      :retry_failure - indicate if failures should be retried (default true)
      :max_retries   - max retries for a failure (default 3)
+     :safe_mode     - indicates if safe mode is used or not (default true)
+     :master_only   - indicates if only redis master is used (default false)
+
+The RedisFailover::Client also supports a custom callback that will be invoked whenever the list of redis clients changes. Example usage:
+
+    RedisFailover::Client.new(:zkservers => 'localhost:2181,localhost:2182,localhost:2183') do |client|
+      client.on_node_change do |master, slaves|
+        logger.info("Nodes changed! master: #{master}, slaves: #{slaves}")
+      end
+    end
 
 ## Manual Failover
 

data/lib/redis_failover/client.rb

@@ -27,52 +27,6 @@ module RedisFailover
     # Amount of time to sleep before retrying a failed operation.
     RETRY_WAIT_TIME = 3
 
-    # Redis read operations that are automatically dispatched to slaves. Any
-    # operation not listed here will be dispatched to the master.
-    REDIS_READ_OPS = Set[
-      :echo,
-      :exists,
-      :get,
-      :getbit,
-      :getrange,
-      :hexists,
-      :hget,
-      :hgetall,
-      :hkeys,
-      :hlen,
-      :hmget,
-      :hvals,
-      :keys,
-      :lindex,
-      :llen,
-      :lrange,
-      :mapped_hmget,
-      :mapped_mget,
-      :mget,
-      :scard,
-      :sdiff,
-      :sinter,
-      :sismember,
-      :smembers,
-      :srandmember,
-      :strlen,
-      :sunion,
-      :type,
-      :zcard,
-      :zcount,
-      :zrange,
-      :zrangebyscore,
-      :zrank,
-      :zrevrange,
-      :zrevrangebyscore,
-      :zrevrank,
-      :zscore
-    ].freeze
-
-    # Unsupported Redis operations. These don't make sense in a client
-    # that abstracts the master/slave servers.
-    UNSUPPORTED_OPS = Set[:select, :dbsize].freeze
-
     # Performance optimization: to avoid unnecessary method_missing calls,
     # we proactively define methods that dispatch to the underlying redis
     # calls.
@@ -93,25 +47,37 @@ module RedisFailover
     # @option options [Logger] :logger logger override
     # @option options [Boolean] :retry_failure indicates if failures are retried
     # @option options [Integer] :max_retries max retries for a failure
+    # @option options [Boolean] :safe_mode indicates if safe mode is used or not
+    # @option options [Boolean] :master_only indicates if only redis master is used
     # @return [RedisFailover::Client]
     def initialize(options = {})
       Util.logger = options[:logger] if options[:logger]
-      @zkservers = options.fetch(:zkservers) { raise ArgumentError, ':zkservers required'}
-      @znode = options[:znode_path] || Util::DEFAULT_ZNODE_PATH
-      @namespace = options[:namespace]
-      @password = options[:password]
-      @db = options[:db]
-      @retry = options[:retry_failure] || true
-      @max_retries = @retry ? options.fetch(:max_retries, 3) : 0
       @master = nil
       @slaves = []
       @node_addresses = {}
       @lock = Monitor.new
       @current_client_key = "current-client-#{self.object_id}"
+      yield self if block_given?
+
+      parse_options(options)
       setup_zk
       build_clients
     end
 
+    # Specifies a callback to invoke when the current redis node list changes.
+    #
+    # @param [Proc] a callback with current master and slaves as arguments
+    #
+    # @example Usage
+    #   RedisFailover::Client.new(:zkservers => zk_servers) do |client|
+    #     client.on_node_change do |master, slaves|
+    #       logger.info("Nodes changed! master: #{master}, slaves: #{slaves}")
+    #     end
+    #   end
+    def on_node_change(&callback)
+      @on_node_change = callback
+    end
+
     # Dispatches redis operations to master/slaves.
     def method_missing(method, *args, &block)
       if redis_operation?(method)
@@ -168,13 +134,31 @@ module RedisFailover
       build_clients
     end
 
+    # Retrieves the current redis master.
+    #
+    # @return [String] the host/port of the current master
+    def current_master
+      master = @lock.synchronize { @master }
+      address_for(master)
+    end
+
+    # Retrieves the current redis slaves.
+    #
+    # @return [Array<String>] an array of known slave host/port addresses
+    def current_slaves
+      slaves = @lock.synchronize { @slaves }
+      addresses_for(slaves)
+    end
+
     private
 
     # Sets up the underlying ZooKeeper connection.
     def setup_zk
       @zk = ZK.new(@zkservers)
       @zk.watcher.register(@znode) { |event| handle_zk_event(event) }
-      @zk.on_expired_session { purge_clients }
+      if @safe_mode
+        @zk.on_expired_session { purge_clients }
+      end
       @zk.on_connected { @zk.stat(@znode, :watch => true) }
       @zk.stat(@znode, :watch => true)
       update_znode_timestamp
@@ -210,7 +194,7 @@ module RedisFailover
     # @param [Proc] block an optional block to pass to the method
     # @return [Object] the result of dispatching the command
     def dispatch(method, *args, &block)
-      unless recently_heard_from_node_manager?
+      if @safe_mode && !recently_heard_from_node_manager?
         build_clients
       end
 
@@ -277,10 +261,26 @@ module RedisFailover
         rescue
           purge_clients
           raise
+        ensure
+          if should_notify?
+           @on_node_change.call(current_master, current_slaves)
+           @last_notified_master = current_master
+           @last_notified_slaves = current_slaves
+          end
         end
       end
     end
 
+    # Determines if the on_node_change callback should be invoked.
+    #
+    # @return [Boolean] true if callback should be invoked, false otherwise
+    def should_notify?
+      return false unless @on_node_change
+      return true if @last_notified_master != current_master
+      return true if different?(Array(@last_notified_slaves), current_slaves)
+      false
+    end
+
     # Fetches the known redis nodes from ZooKeeper.
     #
     # @return [Hash] the known master/slave redis servers
@@ -423,11 +423,19 @@ module RedisFailover
     #   where the same RedisFailover::Client instance is referenced by
     #   nested blocks (e.g., block passed to multi).
     def client_for(method)
-      # stack = Thread.current[@current_client_key] ||= []
-      # client = stack.last || (REDIS_READ_OPS.include?(method) ? slave : master)
-      # stack << client
-      # client
-      master
+      stack = Thread.current[@current_client_key] ||= []
+      client = if stack.last
+        stack.last
+      elsif @master_only
+        master
+      elsif REDIS_READ_OPS.include?(method)
+        slave
+      else
+        master
+      end
+
+      stack << client
+      client
     end
 
     # Pops a client from the thread-local client stack.
@@ -437,5 +445,20 @@ module RedisFailover
       end
       nil
     end
+
+    # Parses the configuration operations.
+    #
+    # @param [Hash] options the configuration options
+    def parse_options(options)
+      @zkservers = options.fetch(:zkservers) { raise ArgumentError, ':zkservers required'}
+      @znode = options.fetch(:znode_path, Util::DEFAULT_ZNODE_PATH)
+      @namespace = options[:namespace]
+      @password = options[:password]
+      @db = options[:db]
+      @retry = options.fetch(:retry_failure, true)
+      @max_retries = @retry ? options.fetch(:max_retries, 3) : 0
+      @safe_mode = options.fetch(:safe_mode, true)
+      @master_only = options.fetch(:master_only, false)
+    end
   end
 end

data/lib/redis_failover/node_manager.rb

@@ -9,14 +9,6 @@ module RedisFailover
   class NodeManager
     include Util
 
-    # Name for the znode that handles exclusive locking between multiple
-    # Node Manager processes. Whoever holds the lock will be considered
-    # the "master" Node Manager, and will be responsible for monitoring
-    # the redis nodes. When a Node Manager that holds the lock disappears
-    # or fails, another Node Manager process will grab the lock and
-    # become the master.
-    LOCK_PATH = 'master_node_manager'
-
     # Number of seconds to wait before retrying bootstrap process.
     TIMEOUT = 5
 
@@ -34,6 +26,14 @@ module RedisFailover
       @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
       @manual_znode = ManualFailover::ZNODE_PATH
       @mutex = Mutex.new
+
+      # Name for the znode that handles exclusive locking between multiple
+      # Node Manager processes. Whoever holds the lock will be considered
+      # the "master" Node Manager, and will be responsible for monitoring
+      # the redis nodes. When a Node Manager that holds the lock disappears
+      # or fails, another Node Manager process will grab the lock and
+      # become the
+      @lock_path = "#{@znode}_lock".freeze
     end
 
     # Starts the node manager.
@@ -44,7 +44,7 @@ module RedisFailover
       @leader = false
       setup_zk
       logger.info('Waiting to become master Node Manager ...')
-      @zk.with_lock(LOCK_PATH) do
+      @zk.with_lock(@lock_path) do
         @leader = true
         logger.info('Acquired master Node Manager lock')
         discover_nodes
@@ -219,13 +219,13 @@ module RedisFailover
     def discover_nodes
       @unavailable = []
       nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
-      raise NoMasterError unless @master = find_master(nodes)
+      @master = find_master(nodes)
       @slaves = nodes - [@master]
       logger.info("Managing master (#{@master}) and slaves" +
         " (#{@slaves.map(&:to_s).join(', ')})")
 
       # ensure that slaves are correctly pointing to this master
-      redirect_slaves_to(@master)
+      redirect_slaves_to(@master) if @master
     end
 
     # Spawns the {RedisFailover::NodeWatcher} instances for each managed node.
@@ -315,7 +315,7 @@ module RedisFailover
     # Creates the znode path containing the redis nodes.
     def create_path
       unless @zk.exists?(@znode)
-        @zk.create(@znode, encode(current_nodes), :ephemeral => true)
+        @zk.create(@znode, encode(current_nodes))
         logger.info("Created ZooKeeper node #{@znode}")
       end
     rescue ZK::Exceptions::NodeExists

data/lib/redis_failover/util.rb

@@ -5,6 +5,52 @@ module RedisFailover
   module Util
     extend self
 
+    # Redis read operations that are automatically dispatched to slaves. Any
+    # operation not listed here will be dispatched to the master.
+    REDIS_READ_OPS = Set[
+      :echo,
+      :exists,
+      :get,
+      :getbit,
+      :getrange,
+      :hexists,
+      :hget,
+      :hgetall,
+      :hkeys,
+      :hlen,
+      :hmget,
+      :hvals,
+      :keys,
+      :lindex,
+      :llen,
+      :lrange,
+      :mapped_hmget,
+      :mapped_mget,
+      :mget,
+      :scard,
+      :sdiff,
+      :sinter,
+      :sismember,
+      :smembers,
+      :srandmember,
+      :strlen,
+      :sunion,
+      :type,
+      :zcard,
+      :zcount,
+      :zrange,
+      :zrangebyscore,
+      :zrank,
+      :zrevrange,
+      :zrevrangebyscore,
+      :zrevrank,
+      :zscore
+    ].freeze
+
+    # Unsupported Redis operations. These don't make sense in a client
+    # that abstracts the master/slave servers.
+    UNSUPPORTED_OPS = Set[:select, :dbsize].freeze
+
     # Default node in ZK that contains the current list of available redis nodes.
     DEFAULT_ZNODE_PATH = '/redis_failover_nodes'.freeze
 
@@ -12,7 +58,7 @@ module RedisFailover
     REDIS_ERRORS = Errno.constants.map { |c| Errno.const_get(c) }
 
     # Connectivity errors that the redis (>3.x) client raises.
-    REDIS_ERRORS << Redis::BaseError if Redis.const_defined?("BaseError")
+    REDIS_ERRORS << Redis::BaseError if Redis.const_defined?('BaseError')
     REDIS_ERRORS.freeze
 
     # Full set of errors related to connectivity.

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = '0.8.9'
+  VERSION = '0.8.10'
 end

data/redis_failover.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = RedisFailover::VERSION
 
-  gem.add_dependency('redis', '~> 3')
+  gem.add_dependency('redis', ['>= 2.2', '< 4'])
   gem.add_dependency('redis-namespace')
   gem.add_dependency('multi_json', '~> 1')
   gem.add_dependency('zk', '~> 1.6')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nogara-redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.8.9
+  version: 0.8.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,30 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-07 00:00:00.000000000 Z
+date: 2012-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - <
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - <
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement