yam-redis-with-retries 2.2.2.1

data/lib/redis/client.rb

@@ -0,0 +1,265 @@
+class Redis
+  class Client
+    attr_accessor :db, :host, :port, :path, :password, :logger
+    attr :timeout
+    attr :connection
+
+    def initialize(options = {})
+      @path = options[:path]
+      if @path.nil?
+        @host = options[:host] || "127.0.0.1"
+        @port = (options[:port] || 6379).to_i
+      end
+
+      @db = (options[:db] || 0).to_i
+      @timeout = (options[:timeout] || 5).to_f
+      @password = options[:password]
+      @logger = options[:logger]
+      @reconnect = true
+      @connection = Connection.drivers.last.new
+    end
+
+    def connect
+      establish_connection
+      call [:auth, @password] if @password
+      call [:select, @db] if @db != 0
+      self
+    end
+
+    def id
+      "redis://#{location}/#{db}"
+    end
+
+    def location
+      @path || "#{@host}:#{@port}"
+    end
+
+    # Starting with 2.2.1, assume that this method is called with a single
+    # array argument. Check its size for backwards compat.
+    def call(*args)
+      if args.first.is_a?(Array) && args.size == 1
+        command = args.first
+      else
+        command = args
+      end
+
+      reply = process([command]) { read }
+      raise reply if reply.is_a?(RuntimeError)
+      reply
+    end
+
+    # Assume that this method is called with a single array argument. No
+    # backwards compat here, since it was introduced in 2.2.2.
+    def call_without_reply(command)
+      process([command])
+      nil
+    end
+
+    # Starting with 2.2.1, assume that this method is called with a single
+    # array argument. Check its size for backwards compat.
+    def call_loop(*args)
+      if args.first.is_a?(Array) && args.size == 1
+        command = args.first
+      else
+        command = args
+      end
+
+      error = nil
+
+      result = without_socket_timeout do
+        process([command]) do
+          loop do
+            reply = read
+            if reply.is_a?(RuntimeError)
+              error = reply
+              break
+            else
+              yield reply
+            end
+          end
+        end
+      end
+
+      # Raise error when previous block broke out of the loop.
+      raise error if error
+
+      # Result is set to the value that the provided block used to break.
+      result
+    end
+
+    def call_pipelined(commands, options = {})
+      options[:raise] = true unless options.has_key?(:raise)
+
+      # The method #ensure_connected (called from #process) reconnects once on
+      # I/O errors. To make an effort in making sure that commands are not
+      # executed more than once, only allow reconnection before the first reply
+      # has been read. When an error occurs after the first reply has been
+      # read, retrying would re-execute the entire pipeline, thus re-issueing
+      # already succesfully executed commands. To circumvent this, don't retry
+      # after the first reply has been read succesfully.
+      first = process(commands) { read }
+      error = first if first.is_a?(RuntimeError)
+
+      begin
+        remaining = commands.size - 1
+        if remaining > 0
+          replies = Array.new(remaining) do
+            reply = read
+            error ||= reply if reply.is_a?(RuntimeError)
+            reply
+          end
+          replies.unshift first
+          replies
+        else
+          replies = [first]
+        end
+      rescue Exception
+        disconnect
+        raise
+      end
+
+      # Raise first error in pipeline when we should raise.
+      raise error if error && options[:raise]
+
+      replies
+    end
+
+    def call_without_timeout(*args)
+      without_socket_timeout do
+        call(*args)
+      end
+    rescue Errno::ECONNRESET
+      retry
+    end
+
+    def process(commands)
+      logging(commands) do
+        ensure_connected do
+          commands.each do |command|
+            connection.write(command)
+          end
+
+          yield if block_given?
+        end
+      end
+    end
+
+    def connected?
+      connection.connected?
+    end
+
+    def disconnect
+      connection.disconnect if connection.connected?
+    end
+
+    def reconnect
+      disconnect
+      connect
+    end
+
+    def read
+      begin
+        connection.read
+
+      rescue Errno::EAGAIN
+        # We want to make sure it reconnects on the next command after the
+        # timeout. Otherwise the server may reply in the meantime leaving
+        # the protocol in a desync status.
+        disconnect
+
+        raise Errno::EAGAIN, "Timeout reading from the socket"
+
+      rescue Errno::ECONNRESET
+        raise Errno::ECONNRESET, "Connection lost"
+      end
+    end
+
+    def without_socket_timeout
+      connect unless connected?
+
+      begin
+        self.timeout = 0
+        yield
+      ensure
+        self.timeout = @timeout if connected?
+      end
+    end
+
+    def without_reconnect
+      begin
+        original, @reconnect = @reconnect, false
+        yield
+      ensure
+        @reconnect = original
+      end
+    end
+
+  protected
+
+    def deprecated(old, new = nil, trace = caller[0])
+      message = "The method #{old} is deprecated and will be removed in 2.0"
+      message << " - use #{new} instead" if new
+      Redis.deprecate(message, trace)
+    end
+
+    def logging(commands)
+      return yield unless @logger && @logger.debug?
+
+      begin
+        commands.each do |name, *args|
+          @logger.debug("Redis >> #{name.to_s.upcase} #{args.join(" ")}")
+        end
+
+        t1 = Time.now
+        yield
+      ensure
+        @logger.debug("Redis >> %0.2fms" % ((Time.now - t1) * 1000))
+      end
+    end
+
+    def establish_connection
+      # Need timeout in usecs, like socket timeout.
+      timeout = Integer(@timeout * 1_000_000)
+
+      if @path
+        connection.connect_unix(@path, timeout)
+      else
+        connection.connect(@host, @port, timeout)
+      end
+
+      # If the timeout is set we set the low level socket options in order
+      # to make sure a blocking read will return after the specified number
+      # of seconds. This hack is from memcached ruby client.
+      self.timeout = @timeout
+
+    rescue Errno::ECONNREFUSED
+      raise Errno::ECONNREFUSED, "Unable to connect to Redis on #{location}"
+    end
+
+    def timeout=(timeout)
+      connection.timeout = Integer(timeout * 1_000_000)
+    end
+
+    def ensure_connected
+      tries = 0
+
+      begin
+        connect unless connected?
+        tries += 1
+
+        yield
+      rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL
+        disconnect
+
+        if tries < 2 && @reconnect
+          retry
+        else
+          raise Errno::ECONNRESET
+        end
+      rescue Exception
+        disconnect
+        raise
+      end
+    end
+  end
+end

data/lib/redis/compat.rb

@@ -0,0 +1,21 @@
+# This file contains core methods that are present in
+# Ruby 1.9 and not in earlier versions.
+
+unless [].respond_to?(:product)
+  class Array
+    def product(*enums)
+      enums.unshift self
+      result = [[]]
+      while [] != enums
+        t, result = result, []
+        b, *enums = enums
+        t.each do |a|
+          b.each do |n|
+            result << a + [n]
+          end
+        end
+      end
+      result
+    end
+  end
+end

data/lib/redis/connection.rb

@@ -0,0 +1,9 @@
+require "redis/connection/registry"
+
+# If a connection driver was required before this file, the array
+# Redis::Connection.drivers will contain one or more classes. The last driver
+# in this array will be used as default driver. If this array is empty, we load
+# the plain Ruby driver as our default. Another driver can be required at a
+# later point in time, causing it to be the last element of the #drivers array
+# and therefore be chosen by default.
+require "redis/connection/ruby" if Redis::Connection.drivers.empty?

data/lib/redis/connection/command_helper.rb

@@ -0,0 +1,45 @@
+class Redis
+  module Connection
+    module CommandHelper
+
+      COMMAND_DELIMITER = "\r\n"
+
+      def build_command(args)
+        command = []
+        command << "*#{args.size}"
+
+        args.each do |arg|
+          arg = arg.to_s
+          command << "$#{string_size arg}"
+          command << arg
+        end
+
+        # Trailing delimiter
+        command << ""
+        command.join(COMMAND_DELIMITER)
+      end
+
+    protected
+
+      if "".respond_to?(:bytesize)
+        def string_size(string)
+          string.to_s.bytesize
+        end
+      else
+        def string_size(string)
+          string.to_s.size
+        end
+      end
+
+      if defined?(Encoding::default_external)
+        def encode(string)
+          string.force_encoding(Encoding::default_external)
+        end
+      else
+        def encode(string)
+          string
+        end
+      end
+    end
+  end
+end

data/lib/redis/connection/hiredis.rb

@@ -0,0 +1,49 @@
+require "redis/connection/registry"
+require "hiredis/connection"
+require "timeout"
+
+class Redis
+  module Connection
+    class Hiredis
+      def initialize
+        @connection = ::Hiredis::Connection.new
+      end
+
+      def connected?
+        @connection.connected?
+      end
+
+      def timeout=(usecs)
+        @connection.timeout = usecs
+      end
+
+      def connect(host, port, timeout)
+        @connection.connect(host, port, timeout)
+      rescue Errno::ETIMEDOUT
+        raise Timeout::Error
+      end
+
+      def connect_unix(path, timeout)
+        @connection.connect_unix(path, timeout)
+      rescue Errno::ETIMEDOUT
+        raise Timeout::Error
+      end
+
+      def disconnect
+        @connection.disconnect
+      end
+
+      def write(command)
+        @connection.write(command)
+      end
+
+      def read
+        @connection.read
+      rescue RuntimeError => err
+        raise ::Redis::ProtocolError.new(err.message)
+      end
+    end
+  end
+end
+
+Redis::Connection.drivers << Redis::Connection::Hiredis

data/lib/redis/connection/registry.rb

@@ -0,0 +1,12 @@
+class Redis
+  module Connection
+
+    # Store a list of loaded connection drivers in the Connection module.
+    # Redis::Client uses the last required driver by default, and will be aware
+    # of the loaded connection drivers if the user chooses to override the
+    # default connection driver.
+    def self.drivers
+      @drivers ||= []
+    end
+  end
+end

data/lib/redis/connection/ruby.rb

@@ -0,0 +1,135 @@
+require "redis/connection/registry"
+require "redis/connection/command_helper"
+require "socket"
+
+class Redis
+  module Connection
+    class Ruby
+      include Redis::Connection::CommandHelper
+
+      MINUS    = "-".freeze
+      PLUS     = "+".freeze
+      COLON    = ":".freeze
+      DOLLAR   = "$".freeze
+      ASTERISK = "*".freeze
+
+      def initialize
+        @sock = nil
+      end
+
+      def connected?
+        !! @sock
+      end
+
+      def connect(host, port, timeout)
+        with_timeout(timeout.to_f / 1_000_000) do
+          @sock = TCPSocket.new(host, port)
+          @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+        end
+      end
+
+      def connect_unix(path, timeout)
+        with_timeout(timeout.to_f / 1_000_000) do
+          @sock = UNIXSocket.new(path)
+        end
+      end
+
+      def disconnect
+        @sock.close
+      rescue
+      ensure
+        @sock = nil
+      end
+
+      def timeout=(usecs)
+        secs   = Integer(usecs / 1_000_000)
+        usecs  = Integer(usecs - (secs * 1_000_000)) # 0 - 999_999
+
+        optval = [secs, usecs].pack("l_2")
+
+        begin
+          @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+          @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+        rescue Errno::ENOPROTOOPT
+        end
+      end
+
+      def write(command)
+        @sock.write(build_command(command))
+      end
+
+      def read
+        # We read the first byte using read() mainly because gets() is
+        # immune to raw socket timeouts.
+        reply_type = @sock.read(1)
+
+        raise Errno::ECONNRESET unless reply_type
+
+        format_reply(reply_type, @sock.gets)
+      end
+
+      def format_reply(reply_type, line)
+        case reply_type
+        when MINUS    then format_error_reply(line)
+        when PLUS     then format_status_reply(line)
+        when COLON    then format_integer_reply(line)
+        when DOLLAR   then format_bulk_reply(line)
+        when ASTERISK then format_multi_bulk_reply(line)
+        else raise ProtocolError.new(reply_type)
+        end
+      end
+
+      def format_error_reply(line)
+        RuntimeError.new(line.strip)
+      end
+
+      def format_status_reply(line)
+        line.strip
+      end
+
+      def format_integer_reply(line)
+        line.to_i
+      end
+
+      def format_bulk_reply(line)
+        bulklen = line.to_i
+        return if bulklen == -1
+        reply = encode(@sock.read(bulklen))
+        @sock.read(2) # Discard CRLF.
+        reply
+      end
+
+      def format_multi_bulk_reply(line)
+        n = line.to_i
+        return if n == -1
+
+        Array.new(n) { read }
+      end
+
+    protected
+
+      begin
+        require "system_timer"
+
+        def with_timeout(seconds, &block)
+          SystemTimer.timeout_after(seconds, &block)
+        end
+
+      rescue LoadError
+        if ! defined?(RUBY_ENGINE)
+          # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
+          # Rubinius should have no issues with timeout.
+          warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Redis client will not hang."
+        end
+
+        require "timeout"
+
+        def with_timeout(seconds, &block)
+          Timeout.timeout(seconds, &block)
+        end
+      end
+    end
+  end
+end
+
+Redis::Connection.drivers << Redis::Connection::Ruby