redis-sentinel 1.1.2 → 1.1.3

data/README.md

@@ -27,6 +27,15 @@ Specify the sentinel servers and master name
 
     Redis.new(master_name: "master1", sentinels: [{host: "localhost", port: 26379}, {host: "localhost", port: 26380}])
 
+There are two additional options: 
+
+1. `:failover_reconnect_timeout` (seconds) will block for that long when
+   redis is unreachable to give failover enough time to take place. Does
+   not wait if not given, or time given is 0.
+
+2. `:failover_reconnect_wait` (seconds) how long to sleep after each
+   failed reconnect during a failover event. Defaults to 0.1s.
+
 ## Example
 
 start redis master server, listen on port 16379
@@ -56,10 +65,22 @@ $ bundle exec ruby example/test.rb
 
 You will see output "bar" every second. Let's try the failover process.
 
-1. stop redis master server
-2. you will see error message output
-3. redis sentinel promote redis slave server to master
-4. then you will see correct "bar" output every second again
+1. Stop redis master server.
+2. You will see error message output.
+3. Redis sentinel promote redis slave server to master. During this time
+   you will see errors instead of "bar" while the failover is happening.
+4. Then you will see correct "bar" output every second again.
+
+## Example of Failover Timeout
+Run the same example code above but run:
+
+```
+$ bundle exec ruby example/test_wait_for_failover.rb
+```
+
+You will see the stream of "bar" will stop while failover is taking
+place and will resume once it has completed, provided that failover
+takes less than 30 seconds.
 
 ## Contributing
 

data/example/test_wait_for_failover.rb

@@ -0,0 +1,19 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ],
+                  :failover_reconnect_timeout => 30)
+redis.set "foo", "bar"
+
+while true
+  begin
+    puts redis.get "foo"
+  rescue => e
+    puts "failover took too long to recover", e
+  end
+  sleep 1
+end

data/lib/redis-sentinel/client.rb

@@ -1,10 +1,16 @@
 require "redis"
 
 class Redis::Client
+  DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS = 0.1
+
   class_eval do
     def initialize_with_sentinel(options={})
-      @master_name = options.delete(:master_name) || options.delete("master_name")
-      @sentinels = options.delete(:sentinels) || options.delete("sentinels")
+      @master_name = fetch_option(options, :master_name)
+      @sentinels = fetch_option(options, :sentinels)
+      @failover_reconnect_timeout = fetch_option(options, :failover_reconnect_timeout)
+      @failover_reconnect_wait = fetch_option(options, :failover_reconnect_wait) ||
+                                 DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS
+
       initialize_without_sentinel(options)
     end
 
@@ -12,8 +18,14 @@ class Redis::Client
     alias initialize initialize_with_sentinel
 
     def connect_with_sentinel
-      discover_master if sentinel?
-      connect_without_sentinel
+      if sentinel?
+        auto_retry_with_timeout do
+          discover_master
+          connect_without_sentinel
+        end
+      else
+        connect_without_sentinel
+      end
     end
 
     alias connect_without_sentinel connect
@@ -23,6 +35,17 @@ class Redis::Client
       @master_name && @sentinels
     end
 
+    def auto_retry_with_timeout(&block)
+      deadline = @failover_reconnect_timeout.to_i + Time.now.to_f
+      begin
+        block.call
+      rescue Redis::CannotConnectError
+        raise if Time.now.to_f > deadline
+        sleep @failover_reconnect_wait
+        retry
+      end
+    end
+
     def try_next_sentinel
       @sentinels << @sentinels.shift
       if @logger && @logger.debug?
@@ -32,10 +55,8 @@ class Redis::Client
     end
 
     def discover_master
-      masters = []
-
       while true
-        sentinel = Redis.new(@sentinels[0])
+        sentinel = redis_sentinels[@sentinels[0]]
 
         begin
           host, port = sentinel.sentinel("get-master-addr-by-name", @master_name)
@@ -50,5 +71,17 @@ class Redis::Client
         end
       end
     end
+
+  private
+
+    def fetch_option(options, key)
+      options.delete(key) || options.delete(key.to_s)
+    end
+
+    def redis_sentinels
+      @redis_sentinels ||= Hash.new do |hash, config|
+        hash[config] = Redis.new(config)
+      end
+    end
   end
 end

data/lib/redis-sentinel/version.rb

@@ -1,5 +1,5 @@
 class Redis
   module Sentinel
-    VERSION = "1.1.2"
+    VERSION = "1.1.3"
   end
 end

data/spec/redis-sentinel/client_spec.rb

@@ -1,6 +1,14 @@
 require "spec_helper"
 
 describe Redis::Client do
+  let(:redis) { mock("Redis", :sentinel => ["remote.server", 8888])}
+
+  subject { Redis::Client.new(:master_name => "master",
+                              :sentinels => [{:host => "localhost", :port => 26379},
+                                             {:host => "localhost", :port => 26380}]) }
+
+  before { Redis.stub(:new).and_return(redis) }
+
   context "#sentinel?" do
     it "should be true if passing sentiels and master_name options" do
       expect(Redis::Client.new(:master_name => "master", :sentinels => [{:host => "localhost", :port => 26379}, {:host => "localhost", :port => 26380}])).to be_sentinel
@@ -20,21 +28,97 @@ describe Redis::Client do
   end
 
   context "#try_next_sentinel" do
-    let(:client) { Redis::Client.new(:master_name => "master", :sentinels => [{:host => "localhost", :port => 26379}, {:host => "localhost", :port => 26380}]) }
-
     it "should return next sentinel server" do
-      expect(client.try_next_sentinel).to eq({:host => "localhost", :port => 26380})
+      expect(subject.try_next_sentinel).to eq({:host => "localhost", :port => 26380})
     end
   end
 
   context "#discover_master" do
-    let(:client) { Redis::Client.new(:master_name => "master", :sentinels => [{:host => "localhost", :port => 26379}, {:host => "localhost", :port => 26380}]) }
-    before { Redis.any_instance.should_receive(:sentinel).with("get-master-addr-by-name", "master").and_return(["remote.server", 8888]) }
+    it "gets the current master" do
+      redis.should_receive(:sentinel).
+            with("get-master-addr-by-name", "master")
+      subject.discover_master
+    end
 
     it "should update options" do
-      client.discover_master
-      expect(client.host).to eq "remote.server"
-      expect(client.port).to eq 8888
+      subject.discover_master
+      expect(subject.host).to eq "remote.server"
+      expect(subject.port).to eq 8888
+    end
+
+    describe "memoizing sentinel connections" do
+      it "does not reconnect to the sentinels" do
+        Redis.should_receive(:new).once
+
+        subject.discover_master
+        subject.discover_master
+      end
+    end
+  end
+
+  context "#auto_retry_with_timeout" do
+    context "no failover reconnect timeout set" do
+      subject { Redis::Client.new }
+
+      it "does not sleep" do
+        subject.should_not_receive(:sleep)
+        expect do
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        end.to raise_error(Redis::CannotConnectError)
+      end
+    end
+
+    context "the failover reconnect timeout is set" do
+      subject { Redis::Client.new(:failover_reconnect_timeout => 3) }
+
+      before(:each) do
+        subject.stub(:sleep)
+      end
+
+      it "only raises after the failover_reconnect_timeout" do
+        called_counter = 0
+        Time.stub(:now).and_return(100, 101, 102, 103, 104, 105)
+
+        begin
+          subject.auto_retry_with_timeout do
+            called_counter += 1
+            raise Redis::CannotConnectError
+          end
+        rescue Redis::CannotConnectError
+        end
+
+        called_counter.should == 4
+      end
+
+      it "sleeps the default wait time" do
+        Time.stub(:now).and_return(100, 101, 105)
+        subject.should_receive(:sleep).with(0.1)
+        begin
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        rescue Redis::CannotConnectError
+        end
+      end
+
+      it "does not catch other errors" do
+        subject.should_not_receive(:sleep)
+        expect do
+          subject.auto_retry_with_timeout { raise Redis::ConnectionError }
+        end.to raise_error(Redis::ConnectionError)
+      end
+
+      context "configured wait time" do
+        subject { Redis::Client.new(:failover_reconnect_timeout => 3,
+                                    :failover_reconnect_wait => 0.01) }
+
+        it "uses the configured wait time" do
+          Time.stub(:now).and_return(100, 101, 105)
+          subject.should_receive(:sleep).with(0.01)
+          begin
+            subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+          rescue Redis::CannotConnectError
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-sentinel
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-20 00:00:00.000000000 Z
+date: 2013-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -61,6 +61,7 @@ files:
 - example/redis-sentinel2.conf
 - example/redis-slave.conf
 - example/test.rb
+- example/test_wait_for_failover.rb
 - lib/redis-sentinel.rb
 - lib/redis-sentinel/client.rb
 - lib/redis-sentinel/version.rb