redis 4.1.0 → 4.5.1

data/lib/redis/cluster/slot.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'set'
-
 class Redis
   class Cluster
     # Keep slot and node key map for Redis Cluster Client
@@ -28,11 +26,20 @@ class Redis
         return nil unless exists?(slot)
         return find_node_key_of_master(slot) if replica_disabled?
 
-        @map[slot][:slaves].to_a.sample
+        @map[slot][:slaves].sample
       end
 
       def put(slot, node_key)
-        assign_node_key(@map, slot, node_key)
+        # Since we're sharing a hash for build_slot_node_key_map, duplicate it
+        # if it already exists instead of preserving as-is.
+        @map[slot] = @map[slot] ? @map[slot].dup : { master: nil, slaves: [] }
+
+        if master?(node_key)
+          @map[slot][:master] = node_key
+        elsif !@map[slot][:slaves].include?(node_key)
+          @map[slot][:slaves] << node_key
+        end
+
         nil
       end
 
@@ -50,19 +57,29 @@ class Redis
         @node_flags[node_key] == ROLE_SLAVE
       end
 
+      # available_slots is mapping of node_key to list of slot ranges
       def build_slot_node_key_map(available_slots)
-        available_slots.each_with_object({}) do |(node_key, slots), acc|
-          slots.each { |slot| assign_node_key(acc, slot, node_key) }
+        by_ranges = {}
+        available_slots.each do |node_key, slots_arr|
+          by_ranges[slots_arr] ||= { master: nil, slaves: [] }
+
+          if master?(node_key)
+            by_ranges[slots_arr][:master] = node_key
+          elsif !by_ranges[slots_arr][:slaves].include?(node_key)
+            by_ranges[slots_arr][:slaves] << node_key
+          end
         end
-      end
 
-      def assign_node_key(mappings, slot, node_key)
-        mappings[slot] ||= { master: nil, slaves: Set.new }
-        if master?(node_key)
-          mappings[slot][:master] = node_key
-        else
-          mappings[slot][:slaves].add(node_key)
+        by_slot = {}
+        by_ranges.each do |slots_arr, nodes|
+          slots_arr.each do |slots|
+            slots.each do |slot|
+              by_slot[slot] = nodes
+            end
+          end
         end
+
+        by_slot
       end
     end
   end

data/lib/redis/cluster/slot_loader.rb

@@ -13,7 +13,7 @@ class Redis
         info = {}
 
         nodes.each do |node|
-          info = Hash[*fetch_slot_info(node)]
+          info = fetch_slot_info(node)
           info.empty? ? next : break
         end
 
@@ -23,9 +23,10 @@ class Redis
       end
 
       def fetch_slot_info(node)
+        hash_with_default_arr = Hash.new { |h, k| h[k] = [] }
         node.call(%i[cluster slots])
-            .map { |arr| parse_slot_info(arr, default_ip: node.host) }
-            .flatten
+            .flat_map { |arr| parse_slot_info(arr, default_ip: node.host) }
+            .each_with_object(hash_with_default_arr) { |arr, h| h[arr[0]] << arr[1] }
       rescue CannotConnectError, ConnectionError, CommandError
         {} # can retry on another node
       end
@@ -34,7 +35,6 @@ class Redis
         first_slot, last_slot = arr[0..1]
         slot_range = (first_slot..last_slot).freeze
         arr[2..-1].map { |addr| [stringify_node_key(addr, default_ip), slot_range] }
-                  .flatten
       end
 
       def stringify_node_key(arr, default_ip)

data/lib/redis/cluster.rb

@@ -78,10 +78,13 @@ class Redis
     end
 
     def call_pipeline(pipeline)
-      node_keys, command_keys = extract_keys_in_pipeline(pipeline)
-      raise CrossSlotPipeliningError, command_keys if node_keys.size > 1
-      node = find_node(node_keys.first)
-      try_send(node, :call_pipeline, pipeline)
+      node_keys = pipeline.commands.map { |cmd| find_node_key(cmd, primary_only: true) }.compact.uniq
+      if node_keys.size > 1
+        raise(CrossSlotPipeliningError,
+              pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?).uniq)
+      end
+
+      try_send(find_node(node_keys.first), :call_pipeline, pipeline)
     end
 
     def call_with_timeout(command, timeout, &block)
@@ -112,12 +115,11 @@ class Redis
       node = Node.new(option.per_node_key)
       available_slots = SlotLoader.load(node)
       node_flags = NodeLoader.load_flags(node)
-      available_node_urls = NodeKey.to_node_urls(available_slots.keys, secure: option.secure?)
-      option.update_node(available_node_urls)
+      option.update_node(available_slots.keys.map { |k| NodeKey.optionize(k) })
       [Node.new(option.per_node_key, node_flags, option.use_replica?),
        Slot.new(available_slots, node_flags, option.use_replica?)]
     ensure
-      node.map(&:disconnect)
+      node&.each(&:disconnect)
     end
 
     def fetch_command_details(nodes)
@@ -128,7 +130,7 @@ class Redis
     def send_command(command, &block)
       cmd = command.first.to_s.downcase
       case cmd
-      when 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
+      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
         @node.call_all(command, &block).first
       when 'flushall', 'flushdb'
         @node.call_master(command, &block).first
@@ -216,9 +218,14 @@ class Redis
       node.public_send(method_name, *args, &block)
     rescue CommandError => err
       if err.message.start_with?('MOVED')
-        assign_redirection_node(err.message).public_send(method_name, *args, &block)
+        raise if retry_count <= 0
+
+        node = assign_redirection_node(err.message)
+        retry_count -= 1
+        retry
       elsif err.message.start_with?('ASK')
         raise if retry_count <= 0
+
         node = assign_asking_node(err.message)
         node.call(%i[asking])
         retry_count -= 1
@@ -226,6 +233,9 @@ class Redis
       else
         raise
       end
+    rescue CannotConnectError
+      update_cluster_info!
+      raise
     end
 
     def assign_redirection_node(err_msg)
@@ -245,14 +255,14 @@ class Redis
       find_node(node_key)
     end
 
-    def find_node_key(command)
+    def find_node_key(command, primary_only: false)
       key = @command.extract_first_key(command)
       return if key.empty?
 
       slot = KeySlotConverter.convert(key)
       return unless @slot.exists?(slot)
 
-      if @command.should_send_to_master?(command)
+      if @command.should_send_to_master?(command) || primary_only
         @slot.find_node_key_of_master(slot)
       else
         @slot.find_node_key_of_slave(slot)
@@ -261,6 +271,7 @@ class Redis
 
     def find_node(node_key)
       return @node.sample if node_key.nil?
+
       @node.find_by(node_key)
     rescue Node::ReloadNeeded
       update_cluster_info!(node_key)
@@ -276,11 +287,5 @@ class Redis
       @node.map(&:disconnect)
       @node, @slot = fetch_cluster_info!(@option)
     end
-
-    def extract_keys_in_pipeline(pipeline)
-      node_keys = pipeline.commands.map { |cmd| find_node_key(cmd) }.compact.uniq
-      command_keys = pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?)
-      [node_keys, command_keys]
-    end
   end
 end

data/lib/redis/connection/command_helper.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 class Redis
   module Connection
     module CommandHelper
-
       COMMAND_DELIMITER = "\r\n"
 
       def build_command(args)
@@ -11,11 +12,13 @@ class Redis
           if i.is_a? Array
             i.each do |j|
               j = j.to_s
+              j = j.encoding == Encoding::BINARY ? j : j.b
               command << "$#{j.bytesize}"
               command << j
             end
           else
             i = i.to_s
+            i = i.encoding == Encoding::BINARY ? i : i.b
             command << "$#{i.bytesize}"
             command << i
           end
@@ -28,7 +31,7 @@ class Redis
         command.join(COMMAND_DELIMITER)
       end
 
-    protected
+      protected
 
       def encode(string)
         string.force_encoding(Encoding.default_external)

data/lib/redis/connection/hiredis.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "registry"
 require_relative "../errors"
 require "hiredis/connection"
@@ -6,7 +8,6 @@ require "timeout"
 class Redis
   module Connection
     class Hiredis
-
       def self.connect(config)
         connection = ::Hiredis::Connection.new
         connect_timeout = (config.fetch(:connect_timeout, 0) * 1_000_000).to_i
@@ -31,7 +32,7 @@ class Redis
       end
 
       def connected?
-        @connection && @connection.connected?
+        @connection&.connected?
       end
 
       def timeout=(timeout)
@@ -57,7 +58,7 @@ class Redis
       rescue Errno::EAGAIN
         raise TimeoutError
       rescue RuntimeError => err
-        raise ProtocolError.new(err.message)
+        raise ProtocolError, err.message
       end
     end
   end

data/lib/redis/connection/registry.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 class Redis
   module Connection
-
     # Store a list of loaded connection drivers in the Connection module.
     # Redis::Client uses the last required driver by default, and will be aware
     # of the loaded connection drivers if the user chooses to override the

data/lib/redis/connection/ruby.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "registry"
 require_relative "command_helper"
 require_relative "../errors"
@@ -13,108 +15,85 @@ end
 class Redis
   module Connection
     module SocketMixin
-
-      CRLF = "\r\n".freeze
+      CRLF = "\r\n"
 
       def initialize(*args)
         super(*args)
 
         @timeout = @write_timeout = nil
-        @buffer = "".dup
+        @buffer = "".b
       end
 
       def timeout=(timeout)
-        if timeout && timeout > 0
-          @timeout = timeout
-        else
-          @timeout = nil
-        end
+        @timeout = (timeout if timeout && timeout > 0)
       end
 
       def write_timeout=(timeout)
-        if timeout && timeout > 0
-          @write_timeout = timeout
-        else
-          @write_timeout = nil
-        end
+        @write_timeout = (timeout if timeout && timeout > 0)
       end
 
       def read(nbytes)
         result = @buffer.slice!(0, nbytes)
 
-        while result.bytesize < nbytes
-          result << _read_from_socket(nbytes - result.bytesize)
-        end
+        buffer = String.new(capacity: nbytes, encoding: Encoding::ASCII_8BIT)
+        result << _read_from_socket(nbytes - result.bytesize, buffer) while result.bytesize < nbytes
 
         result
       end
 
       def gets
-        crlf = nil
-
-        while (crlf = @buffer.index(CRLF)) == nil
-          @buffer << _read_from_socket(1024)
+        while (crlf = @buffer.index(CRLF)).nil?
+          @buffer << _read_from_socket(16_384)
         end
 
         @buffer.slice!(0, crlf + CRLF.bytesize)
       end
 
-      def _read_from_socket(nbytes)
-
-        begin
-          read_nonblock(nbytes)
-
-        rescue IO::WaitReadable
-          if IO.select([self], nil, nil, @timeout)
-            retry
-          else
-            raise Redis::TimeoutError
-          end
-        rescue IO::WaitWritable
-          if IO.select(nil, [self], nil, @timeout)
-            retry
-          else
-            raise Redis::TimeoutError
-          end
-        end
-
-      rescue EOFError
-        raise Errno::ECONNRESET
-      end
-
-      def _write_to_socket(data)
-        begin
-          write_nonblock(data)
-
-        rescue IO::WaitWritable
-          if IO.select(nil, [self], nil, @write_timeout)
-            retry
-          else
-            raise Redis::TimeoutError
-          end
-        rescue IO::WaitReadable
-          if IO.select([self], nil, nil, @write_timeout)
-            retry
-          else
-            raise Redis::TimeoutError
+      def _read_from_socket(nbytes, buffer = nil)
+        loop do
+          case chunk = read_nonblock(nbytes, buffer, exception: false)
+          when :wait_readable
+            unless wait_readable(@timeout)
+              raise Redis::TimeoutError
+            end
+          when :wait_writable
+            unless wait_writable(@timeout)
+              raise Redis::TimeoutError
+            end
+          when nil
+            raise Errno::ECONNRESET
+          when String
+            return chunk
           end
         end
-
-      rescue EOFError
-        raise Errno::ECONNRESET
       end
 
-      def write(data)
-        return super(data) unless @write_timeout
+      def write(buffer)
+        return super(buffer) unless @write_timeout
 
-        length = data.bytesize
-        total_count = 0
+        bytes_to_write = buffer.bytesize
+        total_bytes_written = 0
         loop do
-          count = _write_to_socket(data)
+          case bytes_written = write_nonblock(buffer, exception: false)
+          when :wait_readable
+            unless wait_readable(@write_timeout)
+              raise Redis::TimeoutError
+            end
+          when :wait_writable
+            unless wait_writable(@write_timeout)
+              raise Redis::TimeoutError
+            end
+          when nil
+            raise Errno::ECONNRESET
+          when Integer
+            total_bytes_written += bytes_written
+
+            if total_bytes_written >= bytes_to_write
+              return total_bytes_written
+            end
 
-          total_count += count
-          return total_count if total_count >= length
-          data = data.byteslice(count..-1)
+            buffer = buffer.byteslice(bytes_written..-1)
+          end
         end
       end
     end
@@ -124,7 +103,6 @@ class Redis
       require "timeout"
 
       class TCPSocket < ::TCPSocket
-
         include SocketMixin
 
         def self.connect(host, port, timeout)
@@ -140,7 +118,6 @@ class Redis
       if defined?(::UNIXSocket)
 
         class UNIXSocket < ::UNIXSocket
-
           include SocketMixin
 
           def self.connect(path, timeout)
@@ -152,13 +129,12 @@ class Redis
             raise TimeoutError
           end
 
-          # JRuby raises Errno::EAGAIN on #read_nonblock even when IO.select
+          # JRuby raises Errno::EAGAIN on #read_nonblock even when it
           # says it is readable (1.6.6, in both 1.8 and 1.9 mode).
           # Use the blocking #readpartial method instead.
 
           def _read_from_socket(nbytes)
             readpartial(nbytes)
-
           rescue EOFError
             raise Errno::ECONNRESET
           end
@@ -169,19 +145,16 @@ class Redis
     else
 
       class TCPSocket < ::Socket
-
         include SocketMixin
 
-        def self.connect_addrinfo(ai, port, timeout)
-          sock = new(::Socket.const_get(ai[0]), Socket::SOCK_STREAM, 0)
-          sockaddr = ::Socket.pack_sockaddr_in(port, ai[3])
+        def self.connect_addrinfo(addrinfo, port, timeout)
+          sock = new(::Socket.const_get(addrinfo[0]), Socket::SOCK_STREAM, 0)
+          sockaddr = ::Socket.pack_sockaddr_in(port, addrinfo[3])
 
           begin
             sock.connect_nonblock(sockaddr)
           rescue Errno::EINPROGRESS
-            if IO.select(nil, [sock], nil, timeout) == nil
-              raise TimeoutError
-            end
+            raise TimeoutError unless sock.wait_writable(timeout)
 
             begin
               sock.connect_nonblock(sockaddr)
@@ -220,14 +193,13 @@ class Redis
               return connect_addrinfo(ai, port, timeout)
             rescue SystemCallError
               # Raise if this was our last attempt.
-              raise if addrinfo.length == i+1
+              raise if addrinfo.length == i + 1
             end
           end
         end
       end
 
       class UNIXSocket < ::Socket
-
         include SocketMixin
 
         def self.connect(path, timeout)
@@ -237,9 +209,7 @@ class Redis
           begin
             sock.connect_nonblock(sockaddr)
           rescue Errno::EINPROGRESS
-            if IO.select(nil, [sock], nil, timeout) == nil
-              raise TimeoutError
-            end
+            raise TimeoutError unless sock.wait_writable(timeout)
 
             begin
               sock.connect_nonblock(sockaddr)
@@ -257,18 +227,56 @@ class Redis
       class SSLSocket < ::OpenSSL::SSL::SSLSocket
         include SocketMixin
 
+        unless method_defined?(:wait_readable)
+          def wait_readable(timeout = nil)
+            to_io.wait_readable(timeout)
+          end
+        end
+
+        unless method_defined?(:wait_writable)
+          def wait_writable(timeout = nil)
+            to_io.wait_writable(timeout)
+          end
+        end
+
         def self.connect(host, port, timeout, ssl_params)
           # Note: this is using Redis::Connection::TCPSocket
           tcp_sock = TCPSocket.connect(host, port, timeout)
 
           ctx = OpenSSL::SSL::SSLContext.new
-          ctx.set_params(ssl_params) if ssl_params && !ssl_params.empty?
+
+          # The provided parameters are merged into OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+          ctx.set_params(ssl_params || {})
 
           ssl_sock = new(tcp_sock, ctx)
           ssl_sock.hostname = host
-          ssl_sock.connect
 
-          unless ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
+          begin
+            # Initiate the socket connection in the background. If it doesn't fail
+            # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS)
+            # indicating the connection is in progress.
+            # Unlike waiting for a tcp socket to connect, you can't time out ssl socket
+            # connections during the connect phase properly, because IO.select only partially works.
+            # Instead, you have to retry.
+            ssl_sock.connect_nonblock
+          rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+            if ssl_sock.wait_readable(timeout)
+              retry
+            else
+              raise TimeoutError
+            end
+          rescue IO::WaitWritable
+            if ssl_sock.wait_writable(timeout)
+              retry
+            else
+              raise TimeoutError
+            end
+          end
+
+          unless ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE || (
+            ctx.respond_to?(:verify_hostname) &&
+            !ctx.verify_hostname
+          )
             ssl_sock.post_connection_check(host)
           end
 
@@ -280,15 +288,16 @@ class Redis
     class Ruby
       include Redis::Connection::CommandHelper
 
-      MINUS    = "-".freeze
-      PLUS     = "+".freeze
-      COLON    = ":".freeze
-      DOLLAR   = "$".freeze
-      ASTERISK = "*".freeze
+      MINUS    = "-"
+      PLUS     = "+"
+      COLON    = ":"
+      DOLLAR   = "$"
+      ASTERISK = "*"
 
       def self.connect(config)
         if config[:scheme] == "unix"
           raise ArgumentError, "SSL incompatible with unix sockets" if config[:ssl]
+
           sock = UNIXSocket.connect(config[:path], config[:connect_timeout])
         elsif config[:scheme] == "rediss" || config[:ssl]
           sock = SSLSocket.connect(config[:host], config[:port], config[:connect_timeout], config[:ssl_params])
@@ -300,10 +309,11 @@ class Redis
         instance.timeout = config[:read_timeout]
         instance.write_timeout = config[:write_timeout]
         instance.set_tcp_keepalive config[:tcp_keepalive]
+        instance.set_tcp_nodelay if sock.is_a? TCPSocket
         instance
       end
 
-      if [:SOL_SOCKET, :SO_KEEPALIVE, :SOL_TCP, :TCP_KEEPIDLE, :TCP_KEEPINTVL, :TCP_KEEPCNT].all?{|c| Socket.const_defined? c}
+      if %i[SOL_SOCKET SO_KEEPALIVE SOL_TCP TCP_KEEPIDLE TCP_KEEPINTVL TCP_KEEPCNT].all? { |c| Socket.const_defined? c }
         def set_tcp_keepalive(keepalive)
           return unless keepalive.is_a?(Hash)
 
@@ -315,14 +325,13 @@ class Redis
 
         def get_tcp_keepalive
           {
-            :time   => @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE).int,
-            :intvl  => @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL).int,
-            :probes => @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT).int,
+            time: @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE).int,
+            intvl: @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL).int,
+            probes: @sock.getsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT).int
           }
         end
       else
-        def set_tcp_keepalive(keepalive)
-        end
+        def set_tcp_keepalive(keepalive); end
 
         def get_tcp_keepalive
           {
@@ -330,12 +339,21 @@ class Redis
         end
       end
 
+      # disables Nagle's Algorithm, prevents multiple round trips with MULTI
+      if %i[IPPROTO_TCP TCP_NODELAY].all? { |c| Socket.const_defined? c }
+        def set_tcp_nodelay
+          @sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+        end
+      else
+        def set_tcp_nodelay; end
+      end
+
       def initialize(sock)
         @sock = sock
       end
 
       def connected?
-        !! @sock
+        !!@sock
       end
 
       def disconnect
@@ -346,9 +364,7 @@ class Redis
       end
 
       def timeout=(timeout)
-        if @sock.respond_to?(:timeout=)
-          @sock.timeout = timeout
-        end
+        @sock.timeout = timeout if @sock.respond_to?(:timeout=)
       end
 
       def write_timeout=(timeout)
@@ -363,7 +379,6 @@ class Redis
         line = @sock.gets
         reply_type = line.slice!(0, 1)
         format_reply(reply_type, line)
-
       rescue Errno::EAGAIN
         raise TimeoutError
       end
@@ -375,7 +390,7 @@ class Redis
         when COLON    then format_integer_reply(line)
         when DOLLAR   then format_bulk_reply(line)
         when ASTERISK then format_multi_bulk_reply(line)
-        else raise ProtocolError.new(reply_type)
+        else raise ProtocolError, reply_type
         end
       end
 
@@ -394,6 +409,7 @@ class Redis
       def format_bulk_reply(line)
         bulklen = line.to_i
         return if bulklen == -1
+
         reply = encode(@sock.read(bulklen))
         @sock.read(2) # Discard CRLF.
         reply