redis-sentinel2 1.3.0

data/example/test.rb

@@ -0,0 +1,18 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ])
+redis.set "foo", "bar"
+
+while true
+  begin
+    puts redis.get "foo"
+  rescue => e
+    puts "failed?", e
+  end
+  sleep 1
+end

data/example/test_wait_for_failover.rb

@@ -0,0 +1,19 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ],
+                  :failover_reconnect_timeout => 30)
+redis.set "foo", "bar"
+
+while true
+  begin
+    puts redis.get "foo"
+  rescue => e
+    puts "failover took too long to recover", e
+  end
+  sleep 1
+end

data/example/test_wait_for_failover_write.rb

@@ -0,0 +1,21 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ],
+                  :failover_reconnect_timeout => 30,
+                  :failover_reconnect_wait => 0.0001)
+
+redis.set "foo", 1
+
+while true
+  begin
+    puts redis.incr "foo"
+  rescue Redis::CannotConnectError => e
+    puts "failover took too long to recover", e
+  end
+  sleep 1
+end

data/lib/em-synchrony/redis-sentinel.rb

@@ -0,0 +1,13 @@
+require 'redis/connection/synchrony' unless defined? Redis::Connection::Synchrony
+require 'redis-sentinel'
+
+class Redis::Client
+  class_eval do
+    private
+    def sleep(seconds)
+      f = Fiber.current
+      EM::Timer.new(seconds) { f.resume }
+      Fiber.yield
+    end
+  end
+end

data/lib/redis-sentinel.rb

@@ -0,0 +1,2 @@
+require "redis-sentinel/version"
+require "redis-sentinel/client"

data/lib/redis-sentinel/client.rb

@@ -0,0 +1,121 @@
+require "redis"
+
+class Redis::Client
+  DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS = 0.1
+
+  class_eval do
+    def initialize_with_sentinel(options={})
+      options = options.dup # Don't touch my options
+      @master_name = fetch_option(options, :master_name)
+      @master_password = fetch_option(options, :master_password)
+      @sentinels = fetch_option(options, :sentinels)
+      @failover_reconnect_timeout = fetch_option(options, :failover_reconnect_timeout)
+      @failover_reconnect_wait = fetch_option(options, :failover_reconnect_wait) ||
+                                 DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS
+
+      initialize_without_sentinel(options)
+    end
+
+    alias initialize_without_sentinel initialize
+    alias initialize initialize_with_sentinel
+
+    def connect_with_sentinel
+      if sentinel?
+        auto_retry_with_timeout do
+          discover_master
+          connect_without_sentinel
+        end
+      else
+        connect_without_sentinel
+      end
+    end
+
+    alias connect_without_sentinel connect
+    alias connect connect_with_sentinel
+
+    def sentinel?
+      @master_name && @sentinels
+    end
+
+    def auto_retry_with_timeout(&block)
+      deadline = @failover_reconnect_timeout.to_i + Time.now.to_f
+      begin
+        block.call
+      rescue Redis::CannotConnectError
+        raise if Time.now.to_f > deadline
+        sleep @failover_reconnect_wait
+        retry
+      end
+    end
+
+    def try_next_sentinel
+      @sentinels << @sentinels.shift
+      if @logger && @logger.debug?
+        @logger.debug "Trying next sentinel: #{@sentinels[0][:host]}:#{@sentinels[0][:port]}"
+      end
+      return @sentinels[0]
+    end
+
+    def discover_master
+      while true
+        sentinel = redis_sentinels[@sentinels[0]]
+
+        begin
+          host, port = sentinel.sentinel("get-master-addr-by-name", @master_name)
+          if !host && !port
+            raise Redis::ConnectionError.new("No master named: #{@master_name}")
+          end
+          is_down, runid = sentinel.sentinel("is-master-down-by-addr", host, port)
+          break
+        rescue Redis::CannotConnectError
+          try_next_sentinel
+        end
+      end
+
+      if is_down.to_s == "1" || runid == '?'
+        raise Redis::CannotConnectError.new("The master: #{@master_name} is currently not available.")
+      else
+        self.host = host
+        self.port = port
+        self.password = @master_password
+      end
+    end
+
+    def reconnect_with_sentinels
+      redis_sentinels.each do |config, sentinel|
+        sentinel.client.reconnect
+      end
+      reconnect_without_sentinels
+    end
+
+    alias reconnect_without_sentinels reconnect
+    alias reconnect reconnect_with_sentinels
+
+    def call_with_readonly_protection(*args, &block)
+      tries = 0
+      call_without_readonly_protection(*args, &block)
+    rescue Redis::CommandError => e
+      if e.message == "READONLY You can't write against a read only slave."
+        reconnect
+        retry if (tries += 1) < 4
+      else
+        raise
+      end
+    end
+
+    alias call_without_readonly_protection call
+    alias call call_with_readonly_protection
+
+  private
+
+    def fetch_option(options, key)
+      options.delete(key) || options.delete(key.to_s)
+    end
+
+    def redis_sentinels
+      @redis_sentinels ||= Hash.new do |hash, config|
+        hash[config] = Redis.new(config)
+      end
+    end
+  end
+end

data/lib/redis-sentinel/version.rb

@@ -0,0 +1,5 @@
+class Redis
+  module Sentinel
+    VERSION = "1.3.0"
+  end
+end

data/redis-sentinel.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redis-sentinel/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "redis-sentinel2"
+  gem.version       = Redis::Sentinel::VERSION
+  gem.authors       = ["Richard Huang"]
+  gem.email         = ["flyerhzm@gmail.com"]
+  gem.description   = %q{another redis automatic master/slave failover solution for ruby by using built-in redis sentinel. This version works with redis 2.2.2}
+  gem.summary       = %q{another redis automatic master/slave failover solution for ruby by using built-in redis sentinel. This version works with redis 2.2.2}
+  gem.homepage      = "https://github.com/flyerhzm/redis-sentinel"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "redis", '2.2.2'
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "eventmachine"
+  gem.add_development_dependency "em-synchrony"
+  gem.add_development_dependency "hiredis"
+end

data/spec/redis-sentinel/client_spec.rb

@@ -0,0 +1,187 @@
+require "spec_helper"
+
+describe Redis::Client do
+  let(:client) { double("Client", :reconnect => true) }
+  let(:redis)  { double("Redis", :sentinel => ["remote.server", 8888], :client => client) }
+
+  subject { Redis::Client.new(:master_name => "master", :master_password => "foobar",
+                              :sentinels => [{:host => "localhost", :port => 26379},
+                                             {:host => "localhost", :port => 26380}]) }
+
+  before { Redis.stub(:new).and_return(redis) }
+
+  context "#sentinel?" do
+    it "should be true if passing sentiels and master_name options" do
+      expect(Redis::Client.new(:master_name => "master", :sentinels => [{:host => "localhost", :port => 26379}, {:host => "localhost", :port => 26380}])).to be_sentinel
+    end
+
+    it "should not be true if not passing sentinels and master_name options" do
+      expect(Redis::Client.new).not_to be_sentinel
+    end
+
+    it "should not be true if passing sentinels option but not master_name option" do
+      expect(Redis::Client.new(:sentinels => [{:host => "localhost", :port => 26379}, {:host => "localhost", :port => 26380}])).not_to be_sentinel
+    end
+
+    it "should not be true if passing master_name option but not sentinels option" do
+      expect(Redis::Client.new(:master_name => "master")).not_to be_sentinel
+    end
+  end
+
+  context "#try_next_sentinel" do
+    it "should return next sentinel server" do
+      expect(subject.try_next_sentinel).to eq({:host => "localhost", :port => 26380})
+    end
+  end
+
+  context "#discover_master" do
+    it "gets the current master" do
+      redis.should_receive(:sentinel).
+            with("get-master-addr-by-name", "master")
+      redis.should_receive(:sentinel).
+            with("is-master-down-by-addr", "remote.server", 8888)
+      subject.discover_master
+    end
+
+    it "should update options" do
+      redis.should_receive(:sentinel).
+            with("is-master-down-by-addr", "remote.server", 8888).once.
+            and_return([0, "abc"])
+      subject.discover_master
+      expect(subject.host).to eq "remote.server"
+      expect(subject.port).to eq 8888
+      expect(subject.password).to eq "foobar"
+    end
+
+    it "should not update options before newly promoted master is ready" do
+      redis.should_receive(:sentinel).
+            with("is-master-down-by-addr", "remote.server", 8888).twice.
+            and_return([1, "abc"], [0, "?"])
+      2.times do
+        expect do
+          subject.discover_master
+        end.to raise_error(Redis::CannotConnectError, /currently not available/)
+        expect(subject.host).not_to eq "remote.server"
+        expect(subject.port).not_to eq 8888
+        expect(subject.password).not_to eq "foobar"
+      end
+    end
+
+    it "should not use a password" do
+      Redis.should_receive(:new).with({:host => "localhost", :port => 26379})
+      redis.should_receive(:sentinel).with("get-master-addr-by-name", "master")
+      redis.should_receive(:sentinel).with("is-master-down-by-addr", "remote.server", 8888)
+
+      redis = Redis::Client.new(:master_name => "master", :sentinels => [{:host => "localhost", :port => 26379}])
+      redis.discover_master
+
+      expect(redis.host).to eq "remote.server"
+      expect(redis.port).to eq 8888
+      expect(redis.password).to eq nil
+    end
+
+    it "should select next sentinel" do
+      Redis.should_receive(:new).with({:host => "localhost", :port => 26379})
+      redis.should_receive(:sentinel).
+            with("get-master-addr-by-name", "master").
+            and_raise(Redis::CannotConnectError)
+      Redis.should_receive(:new).with({:host => "localhost", :port => 26380})
+      redis.should_receive(:sentinel).
+            with("get-master-addr-by-name", "master")
+      redis.should_receive(:sentinel).
+            with("is-master-down-by-addr", "remote.server", 8888)
+      subject.discover_master
+      expect(subject.host).to eq "remote.server"
+      expect(subject.port).to eq 8888
+      expect(subject.password).to eq "foobar"
+    end
+
+    describe "memoizing sentinel connections" do
+      it "does not reconnect to the sentinels" do
+        Redis.should_receive(:new).once
+
+        subject.discover_master
+        subject.discover_master
+      end
+    end
+  end
+
+  context "#auto_retry_with_timeout" do
+    context "no failover reconnect timeout set" do
+      subject { Redis::Client.new }
+
+      it "does not sleep" do
+        subject.should_not_receive(:sleep)
+        expect do
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        end.to raise_error(Redis::CannotConnectError)
+      end
+    end
+
+    context "the failover reconnect timeout is set" do
+      subject { Redis::Client.new(:failover_reconnect_timeout => 3) }
+
+      before(:each) do
+        subject.stub(:sleep)
+      end
+
+      it "only raises after the failover_reconnect_timeout" do
+        called_counter = 0
+        Time.stub(:now).and_return(100, 101, 102, 103, 104, 105)
+
+        begin
+          subject.auto_retry_with_timeout do
+            called_counter += 1
+            raise Redis::CannotConnectError
+          end
+        rescue Redis::CannotConnectError
+        end
+
+        called_counter.should == 4
+      end
+
+      it "sleeps the default wait time" do
+        Time.stub(:now).and_return(100, 101, 105)
+        subject.should_receive(:sleep).with(0.1)
+        begin
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        rescue Redis::CannotConnectError
+        end
+      end
+
+      it "does not catch other errors" do
+        subject.should_not_receive(:sleep)
+        expect do
+          subject.auto_retry_with_timeout { raise Redis::ConnectionError }
+        end.to raise_error(Redis::ConnectionError)
+      end
+
+      context "configured wait time" do
+        subject { Redis::Client.new(:failover_reconnect_timeout => 3,
+                                    :failover_reconnect_wait => 0.01) }
+
+        it "uses the configured wait time" do
+          Time.stub(:now).and_return(100, 101, 105)
+          subject.should_receive(:sleep).with(0.01)
+          begin
+            subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+          rescue Redis::CannotConnectError
+          end
+        end
+      end
+    end
+  end
+
+  context "#reconnect" do
+    it "calls reconnect on each sentinel client" do
+      subject.stub(:connect)
+      subject.discover_master
+      subject.send(:redis_sentinels).each do |config, sentinel|
+        sentinel.client.should_receive(:reconnect)
+      end
+
+      subject.reconnect
+    end
+  end
+
+end

data/spec/redis-sentinel/em_client_spec.rb

@@ -0,0 +1,45 @@
+require "spec_helper"
+require "em-synchrony/redis-sentinel"
+require "eventmachine"
+
+describe Redis::Client do
+  context "#auto_retry_with_timeout" do
+    subject { described_class.new(:failover_reconnect_timeout => 3,
+                                :failover_reconnect_wait => 0.1) }
+    context "configured wait time" do
+
+      it "uses the wait time and blocks em" do
+        Time.stub(:now).and_return(100, 101, 105)
+        flag = false; EM.next_tick { flag = true }
+        subject.should_receive(:sleep).with(0.1).and_return(0.1)
+        begin
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        rescue Redis::CannotConnectError
+        end
+        flag.should be_false
+      end
+
+      it "uses the wait time and doesn't block em" do
+        Time.stub(:now).and_return(100, 101, 105)
+        flag = false; EM.next_tick { flag = true }
+        begin
+          subject.auto_retry_with_timeout { raise Redis::CannotConnectError }
+        rescue Redis::CannotConnectError
+        end
+        flag.should be_true
+      end
+    end
+  end
+
+  around(:each) do |testcase|
+    EM.run do
+      Fiber.new do
+        begin
+          testcase.call
+        ensure
+          EM.stop
+        end
+      end.resume
+    end
+  end
+end