redic-sentinel 1.5.1

data/example/test.rb

@@ -0,0 +1,18 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ])
+redis.set "foo", "bar"
+
+while true
+  begin
+    puts redis.get "foo"
+  rescue => e
+    puts "failed?", e
+  end
+  sleep 1
+end

data/example/test_wait_for_failover.rb

@@ -0,0 +1,19 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ],
+                  :failover_reconnect_timeout => 30)
+redis.set "foo", "bar"
+
+while true
+  begin
+    puts redis.get "foo"
+  rescue => e
+    puts "failover took too long to recover", e
+  end
+  sleep 1
+end

data/example/test_wait_for_failover_write.rb

@@ -0,0 +1,21 @@
+require 'redis'
+require 'redis-sentinel'
+
+redis = Redis.new(:master_name => "example-test",
+                  :sentinels => [
+                    {:host => "localhost", :port => 26379},
+                    {:host => "localhost", :port => 26380}
+                  ],
+                  :failover_reconnect_timeout => 30,
+                  :failover_reconnect_wait => 0.0001)
+
+redis.set "foo", 1
+
+while true
+  begin
+    puts redis.incr "foo"
+  rescue Redis::CannotConnectError => e
+    puts "failover took too long to recover", e
+  end
+  sleep 1
+end

data/lib/redic-sentinel.rb

@@ -0,0 +1,3 @@
+require "redic-sentinel/version"
+require "redic-sentinel/client"
+require "redic-sentinel/redic"

data/lib/redic-sentinel/client.rb

@@ -0,0 +1,188 @@
+require "redic"
+require "redis"
+
+class Redic::Client
+  DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS = 0.1
+
+  class_eval do
+    attr_reader :current_sentinel
+    attr_reader :uri
+
+    def initialize_with_sentinel(url, timeout, options={})
+      options = options.dup # Don't touch my options
+      @options = options
+      @master_name = fetch_option(options, :master_name)
+      @master_password = fetch_option(options, :master_password)
+      @sentinels_options = _parse_sentinel_options(fetch_option(options, :sentinels))
+      @failover_reconnect_timeout = fetch_option(options, :failover_reconnect_timeout)
+      @failover_reconnect_wait = fetch_option(options, :failover_reconnect_wait) ||
+                                 DEFAULT_FAILOVER_RECONNECT_WAIT_SECONDS
+
+      Thread.new { watch_sentinel } if sentinel? && !fetch_option(options, :async)
+
+      initialize_without_sentinel(url, timeout)
+    end
+
+    alias initialize_without_sentinel initialize
+    alias initialize initialize_with_sentinel
+
+    def establish_connection_with_sentinel
+      if sentinel?
+        auto_retry_with_timeout do
+          discover_master
+          establish_connection_without_sentinel
+        end
+      else
+        establish_connection_without_sentinel
+      end
+    end
+
+    alias establish_connection_without_sentinel establish_connection
+    alias establish_connection establish_connection_with_sentinel
+
+    def sentinel?
+      !!(@master_name && @sentinels_options)
+    end
+
+    def auto_retry_with_timeout(&block)
+      deadline = @failover_reconnect_timeout.to_i + Time.now.to_f
+      begin
+        block.call
+      rescue Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH => e
+        raise if Time.now.to_f > deadline
+        sleep @failover_reconnect_wait
+        retry
+      end
+    end
+
+    def new_sentinel(sentinel_options)
+      Redis.new(sentinel_options)
+    end
+
+    def try_next_sentinel
+      sentinel_options = @sentinels_options.shift
+      @sentinels_options.push sentinel_options
+
+      @logger.debug "Trying next sentinel: #{sentinel_options[:host]}:#{sentinel_options[:port]}" if @logger && @logger.debug?
+      @current_sentinel = new_sentinel(sentinel_options)
+    end
+
+    def refresh_sentinels_list
+      current_sentinel.sentinel("sentinels", @master_name).each do |response|
+        @sentinels_options << {:host => response[3], :port => response[5]}
+      end
+      @sentinels_options.uniq! {|h| h.values_at(:host, :port) }
+    end
+
+    def switch_master(host, port)
+      @uri = URI.parse("redis://#{host}:#{port}/")
+    end
+
+    def discover_master
+      attempts = 0
+      while true
+        attempts += 1
+        try_next_sentinel
+
+        begin
+          master_host, master_port = current_sentinel.sentinel("get-master-addr-by-name", @master_name)
+          if master_host && master_port
+            # An ip:port pair
+            switch_master(master_host, master_port)
+            refresh_sentinels_list
+            break
+          end
+        rescue Redis::CommandError => e
+          raise e unless e.message.include?("IDONTKNOW")
+        rescue Redis::CannotConnectError, Redis::ConnectionError, Errno::EHOSTDOWN, Errno::EHOSTUNREACH => e
+          # failed to connect to current sentinel server
+        end
+
+        raise "Cannot connect to master (too many attempts)" if attempts > @sentinels_options.count
+      end
+    end
+
+    def call_with_readonly_protection(*args, &block)
+      readonly_protection_with_timeout(:call_without_readonly_protection, *args, &block)
+    end
+
+    alias call_without_readonly_protection call
+    alias call call_with_readonly_protection
+
+    def watch_sentinel
+      while true
+        puts "Acquire new sentinel"
+        sentinel = new_sentinel(@sentinels_options[0])
+
+        begin
+          puts "Subscribe sentinel"
+          sentinel.psubscribe("*") do |on|
+            on.pmessage do |pattern, channel, message|
+              puts "New message"
+              puts "#{pattern} #{channel} #{message}"
+              next if channel != "+switch-master"
+
+              master_name, old_host, old_port, new_host, new_port = message.split(" ")
+
+              next if master_name != @master_name
+
+              switch_master(new_host, new_port)
+
+              @logger.debug "Failover: #{old_host}:#{old_port} => #{new_host}:#{new_port}" if @logger && @logger.debug?
+
+              @connection = nil
+            end
+          end
+        rescue Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH
+          puts "Cannot connect to sentinel"
+          try_next_sentinel
+          sleep 1
+        end
+      end
+    end
+
+  private
+    def reconnect
+      @connection = nil
+      connect {}
+    end
+
+    def readonly_protection_with_timeout(method, *args, &block)
+      deadline = @failover_reconnect_timeout.to_i + Time.now.to_f
+      send(method, *args, &block)
+    rescue StandardError => e
+      if e.message.include? "READONLY You can't write against a read only slave."
+        reconnect
+        raise if Time.now.to_f > deadline
+        sleep @failover_reconnect_wait
+        retry
+      else
+        raise
+      end
+    end
+
+    def fetch_option(options, key)
+      options.delete(key) || options.delete(key.to_s)
+    end
+
+    def _parse_sentinel_options(options)
+      return if options.nil?
+
+      sentinel_options = []
+      options.each do |opts|
+        opts = opts[:url] if opts.is_a?(Hash) && opts.key?(:url)
+        case opts
+        when Hash
+          sentinel_options << opts
+        else
+          uri = URI.parse(opts)
+          sentinel_options << {
+            :host => uri.host,
+            :port => uri.port
+          }
+        end
+      end
+      sentinel_options
+    end
+  end
+end

data/lib/redic-sentinel/redic.rb

@@ -0,0 +1,9 @@
+require "redic"
+
+class Redic
+  def initialize(url = "redis://127.0.0.1:6379", timeout = 10_000_000, options = {})
+    @url = url
+    @client = Redic::Client.new(url, timeout, options)
+    @queue = []
+  end
+end

data/lib/redic-sentinel/version.rb

@@ -0,0 +1,5 @@
+class Redic
+  module Sentinel
+    VERSION = "1.5.1"
+  end
+end

data/redic-sentinel.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redic-sentinel/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "redic-sentinel"
+  gem.version       = Redic::Sentinel::VERSION
+  gem.authors       = ["Richard Huang", "Phuong Gia Su"]
+  gem.email         = ["flyerhzm@gmail.com", "phuongnd08@gmail.com"]
+  gem.description   = %q{automatic master/slave failover solution for redic by using built-in redis sentinel}
+  gem.summary       = %q{automatic master/slave failover solution for redic by using built-in redis sentinel}
+  gem.homepage      = "https://github.com/phuongnd08/redic-sentinel"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "redic"
+  gem.add_dependency "redis"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "eventmachine"
+  gem.add_development_dependency "em-synchrony"
+  gem.add_development_dependency "hiredis"
+end

data/spec/redic-sentinel/client_spec.rb

@@ -0,0 +1,196 @@
+require "spec_helper"
+
+describe Redic::Client do
+  let(:client) { double("Client", :reconnect => true) }
+  let(:current_sentinel)  { double("Redis", :client => client) }
+
+  let(:sentinels) do
+    [
+      { :host => "localhost", :port => 26379 },
+      'sentinel://localhost:26380',
+      { :url => 'sentinel://localhost:26381' },
+    ]
+  end
+
+  let(:slaves_reply) do
+    [
+      ["ip", "slave-0", "port", "6379"],
+      ["ip", "slave-1", "port", "6380"],
+      ["ip", "slave-2", "port", "6381"]
+    ]
+  end
+
+  subject { Redic::Client.new("redis://test.host:6379/", 10_000_000, :master_name => "master", :master_password => "foobar",
+                              :sentinels => sentinels) }
+
+  context "new instances" do
+    it "should parse sentinel options" do
+      expect(subject.instance_variable_get(:@sentinels_options)).to match_array [
+        {:host=>"localhost", :port=>26379},
+        {:host=>"localhost", :port=>26380},
+        {:host=>"localhost", :port=>26381}
+      ]
+    end
+  end
+
+  context "#sentinel?" do
+    it "should be true if passing sentiels and master_name options" do
+      expect(subject).to be_sentinel
+    end
+
+    it "should not be true if not passing sentinels and master_name options" do
+      expect(Redic::Client.new("redis://test.host:6379/", 10_000_000)).not_to be_sentinel
+    end
+
+    it "should not be true if passing sentinels option but not master_name option" do
+      client = Redic::Client.new(
+        "redis://test.host:6379/", 10_000_000,
+        :sentinels => [
+          {:host => "localhost", :port => 26379},
+          {:host => "localhost", :port => 26380}
+        ])
+      expect(client).not_to be_sentinel
+    end
+
+    it "should not be true if passing master_name option but not sentinels option" do
+      client = Redic::Client.new("redis://test.host:6379/", 10_000_000, :master_name => "master")
+      expect(client).not_to be_sentinel
+    end
+
+    it "should be true if passing master_name, and sentinels as uri" do
+      client = Redic::Client.new("redis://test.host:6379/", 10_000_000, :master_name => "master",
+        :sentinels => %w(sentinel://localhost:26379 sentinel://localhost:26380))
+      expect(client).to be_sentinel
+    end
+  end
+
+  context "#try_next_sentinel" do
+    before { allow(Redis).to receive(:new).and_return(current_sentinel) }
+
+    it "returns next sentinel server" do
+      expect(subject.try_next_sentinel).to eq current_sentinel
+    end
+  end
+
+  context "#refresh_sentinels_list" do
+    it "gets all sentinels list" do
+      allow(subject).to receive(:current_sentinel).and_return(current_sentinel)
+      expect(current_sentinel).to receive(:sentinel).with("sentinels", "master").and_return([
+        ["name", "localhost:26381", "ip", "localhost", "port", 26380],
+        ["name", "localhost:26381", "ip", "localhost", "port", 26381],
+        ["name", "localhost:26381", "ip", "localhost", "port", 26382],
+      ])
+      subject.refresh_sentinels_list
+      expect(subject.instance_variable_get(:@sentinels_options)).to eq [
+        {:host => "localhost", :port => 26379},
+        {:host => "localhost", :port => 26380},
+        {:host => "localhost", :port => 26381},
+        {:host => "localhost", :port => 26382},
+      ]
+    end
+  end
+
+  context "#discover_master" do
+    before do
+      allow(subject).to receive(:try_next_sentinel)
+      allow(subject).to receive(:refresh_sentinels_list)
+      allow(subject).to receive(:current_sentinel).and_return(current_sentinel)
+    end
+
+    it "updates master config options" do
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_return(["master", 8888])
+      subject.discover_master
+      expect(subject.uri.to_s).to eq "redis://master:8888/"
+    end
+
+    it "selects next sentinel if failed to connect to current_sentinel" do
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_raise(Redis::ConnectionError)
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_return(["master", 8888])
+      subject.discover_master
+      expect(subject.uri.to_s).to eq "redis://master:8888/"
+    end
+
+    it "selects next sentinel if sentinel doesn't know" do
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_raise(Redis::CommandError.new("IDONTKNOW: No idea"))
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_return(["master", 8888])
+      subject.discover_master
+      expect(subject.uri.to_s).to eq "redis://master:8888/"
+    end
+
+    it "raises error if try_next_sentinel raises command error" do
+      expect(current_sentinel).to receive(:sentinel).with("get-master-addr-by-name", "master").and_raise(Redis::CommandError)
+      expect { subject.discover_master }.to raise_error(Redis::CommandError)
+    end
+
+    it "raises error if try_next_sentinel raises connection error" do
+      expect(subject).to receive(:try_next_sentinel).and_raise(Redis::CannotConnectError)
+      expect { subject.discover_master }.to raise_error(Redis::CannotConnectError)
+    end
+  end
+
+  context "#auto_retry_with_timeout" do
+    context "no failover reconnect timeout set" do
+      subject { Redic::Client.new("redis://test.host:6379/", 10_000_000) }
+
+      it "does not sleep" do
+        expect(subject).not_to receive(:sleep)
+        expect {
+          subject.auto_retry_with_timeout { raise Errno::ECONNREFUSED }
+        }.to raise_error(Errno::ECONNREFUSED)
+      end
+    end
+
+    context "the failover reconnect timeout is set" do
+      subject { Redic::Client.new("redis://test.host:6379/", 10_000_000, :failover_reconnect_timeout => 3) }
+
+      before(:each) do
+        allow(subject).to receive(:sleep)
+      end
+
+      it "only raises after the failover_reconnect_timeout" do
+        called_counter = 0
+        allow(Time).to receive(:now).and_return(100, 101, 102, 103, 104, 105)
+
+        begin
+          subject.auto_retry_with_timeout do
+            called_counter += 1
+            raise Errno::ECONNREFUSED
+          end
+        rescue Errno::ECONNREFUSED
+        end
+
+        expect(called_counter).to eq(4)
+      end
+
+      it "sleeps the default wait time" do
+        allow(Time).to receive(:now).and_return(100, 101, 105)
+        expect(subject).to receive(:sleep).with(0.1)
+        begin
+          subject.auto_retry_with_timeout { raise Errno::ECONNREFUSED }
+        rescue Errno::ECONNREFUSED
+        end
+      end
+
+      it "does not catch other errors" do
+        expect(subject).not_to receive(:sleep)
+        expect do
+          subject.auto_retry_with_timeout { raise "Some other error" }
+        end.to raise_error("Some other error")
+      end
+
+      context "configured wait time" do
+        subject { Redic::Client.new("redis://test.host:6379/", 10_000_000, :failover_reconnect_timeout => 3,
+                                    :failover_reconnect_wait => 0.01) }
+
+        it "uses the configured wait time" do
+          allow(Time).to receive(:now).and_return(100, 101, 105)
+          expect(subject).to receive(:sleep).with(0.01)
+          begin
+            subject.auto_retry_with_timeout { raise Errno::ECONNREFUSED }
+          rescue Errno::ECONNREFUSED
+          end
+        end
+      end
+    end
+  end
+end