redis-client 0.5.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b23930fc431afe23313aca3ef90ba49f219a6ed1a50f8153075a09b4c43678c0
-  data.tar.gz: 502388c79068a1dfe166543a6eb91cead665f42e404f679a598f5a8e2844ee82
+  metadata.gz: 2b3a285f6dbfb718f7fa429e962dbb955f3109cf67d785869076c8f0407d7d45
+  data.tar.gz: e7e56c056e9eaabc96e438474c70bd2a3ff46a386c28cc891c4b13acbd255492
 SHA512:
-  metadata.gz: 9f6d4422b8c3e6608a5a8c53a55335ad0c44c034bc2cf26c7b5aace8dfdbe799409dd9783bd5928da3453d60ed28579594676f95c951a6a4dfff5e04dd09d28d
-  data.tar.gz: 3ee257ab6d509606c959079a16886389d76bfe232876aba9de1ae8922031746ad2576224fc5b0fae44a09ef13ba4d14ea862098860f923f432f63454499c1677
+  metadata.gz: 0f70dfa045ca15360ba86f18cfa278a98296531b20b661a39abc8788091ff34f5407ec084e48fac45e1a2362f9741512ab35bfcc93d3fd16b3170fd188c57373
+  data.tar.gz: eee07f8f34ba002f46201e50d2d72ff6f05192ccaa06ae23e952bde1008f2b33783afda48ac37ad70d512619395ac442d129d305adf78c17eced954edd9922fa

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Unreleased
 
+# 0.6.2
+
+- Fix sentinel to not connected to s_down or o_down replicas.
+
+# 0.6.1
+
+- Fix `REDIS_REPLY_SET` parsing in `hiredis`.
+
+# 0.6.0
+
+- Added `protocol: 2` options to talk with Redis 5 and older servers.
+- Added `_v` versions of `call` methods to make it easier to pass commands as arrays without splating.
+- Fix calling `blocking_call` with a block in a pipeline.
+- `blocking_call` now raise `ReadTimeoutError` if the command didn't complete in time.
+- Fix `blocking_call` to not respect `retry_attempts` on timeout.
+- Stop parsing RESP3 sets as Ruby Set instances.
+- Fix `SystemStackError` when parsing very large hashes. Fix: #30
+- `hiredis` now more properly release the GVL when doing IOs.
+
 # 0.5.1
 
 - Fix a regression in the `scan` familly of methods, they would raise with `ArgumentError: can't issue an empty redis command`. Fix: #24

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    redis-client (0.5.1)
+    redis-client (0.6.2)
       connection_pool
 
 GEM

data/README.md

@@ -81,6 +81,7 @@ redis.call("GET", "mykey")
 - `read_timeout`: The read timeout, takes precedence over the general timeout when reading responses from the server.
 - `write_timeout`: The write timeout, takes precedence over the general timeout when sending commands to the server.
 - `reconnect_attempts`: Specify how many times the client should retry to send queries. Defaults to `0`. Makes sure to read the [reconnection section](#reconnection) before enabling it.
+- `protocol:` The version of the RESP protocol to use. Default to `3`.
 
 ### Sentinel support
 
@@ -207,6 +208,16 @@ redis.call("EXISTS", "counter") # => 1
 redis.call("EXISTS", "counter") { |c| c > 0 } # => true
 ```
 
+### `*_v` methods
+
+In some it's more convenient to pass commands as arrays, for that `_v` versions of `call` methods are available.
+
+```ruby
+redis.call_v(["MGET"] + keys)
+redis.blocking_call_v(1, ["MGET"] + keys)
+redis.call_once_v(1, ["MGET"] + keys)
+```
+
 ### Blocking commands
 
 For blocking commands such as `BRPOP`, a custom timeout duration can be passed as first argument of the `#blocking_call` method:
@@ -215,7 +226,7 @@ For blocking commands such as `BRPOP`, a custom timeout duration can be passed a
 redis.blocking_call(timeout, "BRPOP", "key", 0)
 ```
 
-If `timeout` is reached, `#blocking_call` returns `nil`.
+If `timeout` is reached, `#blocking_call` raises `RedisClient::ReadTimeoutError` and doesn't retry regardless of the `reconnect_attempts` configuration.
 
 `timeout` is expressed in seconds, you can pass `false` or `0` to mean no timeout.
 

data/Rakefile

@@ -107,8 +107,8 @@ end
 
 if hiredis_supported
   task default: %i[compile test rubocop]
-  task ci: %i[compile test]
+  task ci: %i[compile test:ruby test:hiredis]
 else
   task default: %i[test rubocop]
-  task ci: %i[test]
+  task ci: %i[test:ruby]
 end

data/lib/redis_client/command_builder.rb

@@ -5,19 +5,17 @@ class RedisClient
     extend self
 
     if Symbol.method_defined?(:name)
-      def generate!(args, kwargs)
+      def generate(args, kwargs = nil)
         command = args.flat_map do |element|
           case element
           when Hash
             element.flatten
-          when Set
-            element.to_a
           else
             element
           end
         end
 
-        kwargs.each do |key, value|
+        kwargs&.each do |key, value|
           if value
             if value == true
               command << key.name
@@ -47,19 +45,17 @@ class RedisClient
         command
       end
     else
-      def generate!(args, kwargs)
+      def generate(args, kwargs = nil)
         command = args.flat_map do |element|
           case element
           when Hash
             element.flatten
-          when Set
-            element.to_a
           else
             element
           end
         end
 
-        kwargs.each do |key, value|
+        kwargs&.each do |key, value|
           if value
             if value == true
               command << key.to_s

data/lib/redis_client/config.rb

@@ -12,8 +12,8 @@ class RedisClient
     DEFAULT_DB = 0
 
     module Common
-      attr_reader :db, :username, :password, :id, :ssl, :ssl_params, :command_builder,
-        :connect_timeout, :read_timeout, :write_timeout, :driver, :connection_prelude
+      attr_reader :db, :password, :id, :ssl, :ssl_params, :command_builder,
+        :connect_timeout, :read_timeout, :write_timeout, :driver, :connection_prelude, :protocol
 
       alias_method :ssl?, :ssl
 
@@ -29,10 +29,12 @@ class RedisClient
         ssl: nil,
         ssl_params: nil,
         driver: nil,
+        protocol: 3,
+        client_implementation: RedisClient,
         command_builder: CommandBuilder,
         reconnect_attempts: false
       )
-        @username = username || DEFAULT_USERNAME
+        @username = username
         @password = password
         @db = db || DEFAULT_DB
         @id = id
@@ -45,14 +47,23 @@ class RedisClient
 
         @driver = driver ? RedisClient.driver(driver) : RedisClient.default_driver
 
+        @client_implementation = client_implementation
+        @protocol = protocol
+        unless protocol == 2 || protocol == 3
+          raise ArgumentError, "Unknown protocol version #{protocol.inspect}, expected 2 or 3"
+        end
+
         @command_builder = command_builder
 
         reconnect_attempts = Array.new(reconnect_attempts, 0).freeze if reconnect_attempts.is_a?(Integer)
         @reconnect_attempts = reconnect_attempts
-
         @connection_prelude = build_connection_prelude
       end
 
+      def username
+        @username || DEFAULT_USERNAME
+      end
+
       def sentinel?
         false
       end
@@ -63,7 +74,7 @@ class RedisClient
       end
 
       def new_client(**kwargs)
-        RedisClient.new(self, **kwargs)
+        @client_implementation.new(self, **kwargs)
       end
 
       def retry_connecting?(attempt, _error)
@@ -79,7 +90,7 @@ class RedisClient
       end
 
       def ssl_context
-        @ssl_context ||= @driver.ssl_context(@ssl_params)
+        @ssl_context ||= @driver.ssl_context(@ssl_params || {})
       end
 
       def server_url
@@ -94,10 +105,18 @@ class RedisClient
 
       def build_connection_prelude
         prelude = []
-        prelude <<  if @password
-          ["HELLO", "3", "AUTH", @username, @password]
-        else
-          ["HELLO", "3"]
+        if protocol == 3
+          prelude << if @password
+            ["HELLO", "3", "AUTH", @username || DEFAULT_USERNAME, @password]
+          else
+            ["HELLO", "3"]
+          end
+        elsif @password
+          prelude << if @username && !@username.empty?
+            ["AUTH", @username, @password]
+          else
+            ["AUTH", @password]
+          end
         end
 
         if @db && @db != 0
@@ -136,7 +155,7 @@ class RedisClient
       super(**kwargs)
 
       @host = host || uri&.host&.sub(/\A\[(.*)\]\z/, '\1') || DEFAULT_HOST
-      @port = port || uri&.port || DEFAULT_PORT
+      @port = Integer(port || uri&.port || DEFAULT_PORT)
       @path = path
     end
   end

data/lib/redis_client/decorator.rb

@@ -20,7 +20,7 @@ class RedisClient
         @client = client
       end
 
-      %i(call call_once blocking_call).each do |method|
+      %i(call call_v call_once call_once_v blocking_call blocking_call_v).each do |method|
         class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
           def #{method}(*args, &block)
             @client.#{method}(*args, &block)
@@ -64,7 +64,7 @@ class RedisClient
         RUBY
       end
 
-      %i(id config size connect_timeout read_timeout write_timeout).each do |reader|
+      %i(id config size connect_timeout read_timeout write_timeout pubsub).each do |reader|
         class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
           def #{reader}
             @client.#{reader}

data/lib/redis_client/pooled.rb

@@ -49,7 +49,7 @@ class RedisClient
       pool.size
     end
 
-    methods = %w(pipelined multi pubsub call call_once blocking_call)
+    methods = %w(pipelined multi pubsub call call_v call_once call_once_v blocking_call blocking_call_v)
     iterable_methods = %w(scan sscan hscan zscan)
     begin
       methods.each do |method|

data/lib/redis_client/ruby_connection/resp3.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 class RedisClient
   module RESP3
     module_function
@@ -41,8 +39,6 @@ class RedisClient
         case element
         when Hash
           element.flatten
-        when Set
-          element.to_a
         else
           element
         end
@@ -55,7 +51,7 @@ class RedisClient
     end
 
     def new_buffer
-      String.new(encoding: Encoding::BINARY, capacity: 128)
+      String.new(encoding: Encoding::BINARY, capacity: 127)
     end
 
     def dump_any(object, buffer)
@@ -144,11 +140,15 @@ class RedisClient
     end
 
     def parse_set(io)
-      parse_sequence(io, parse_integer(io)).to_set
+      parse_sequence(io, parse_integer(io))
     end
 
     def parse_map(io)
-      Hash[*parse_sequence(io, parse_integer(io) * 2)]
+      hash = {}
+      parse_integer(io).times do
+        hash[parse(io)] = parse(io)
+      end
+      hash
     end
 
     def parse_push(io)
@@ -156,6 +156,8 @@ class RedisClient
     end
 
     def parse_sequence(io, size)
+      return if size < 0 # RESP2 nil
+
       array = Array.new(size)
       size.times do |index|
         array[index] = parse(io)
@@ -185,6 +187,8 @@ class RedisClient
 
     def parse_blob(io)
       bytesize = parse_integer(io)
+      return if bytesize < 0 # RESP2 nil type
+
       str = io.read_chomp(bytesize)
       str.force_encoding(Encoding.default_external)
       str.force_encoding(Encoding::BINARY) unless str.valid_encoding?

data/lib/redis_client/ruby_connection.rb

@@ -60,9 +60,9 @@ class RedisClient
         loop do
           case status = socket.connect_nonblock(exception: false)
           when :wait_readable
-            socket.to_io.wait_readable(connect_timeout) or raise ReadTimeoutError
+            socket.to_io.wait_readable(connect_timeout) or raise CannotConnectError
           when :wait_writable
-            socket.to_io.wait_writable(connect_timeout) or raise WriteTimeoutError
+            socket.to_io.wait_writable(connect_timeout) or raise CannotConnectError
           when socket
             break
           else
@@ -76,10 +76,8 @@ class RedisClient
         read_timeout: read_timeout,
         write_timeout: write_timeout,
       )
-    rescue Errno::ETIMEDOUT => error
-      raise ConnectTimeoutError, error.message
     rescue SystemCallError, OpenSSL::SSL::SSLError, SocketError => error
-      raise ConnectionError, error.message
+      raise CannotConnectError, error.message, error.backtrace
     end
 
     def connected?
@@ -125,6 +123,8 @@ class RedisClient
       else
         @io.with_timeout(timeout) { RESP3.load(@io) }
       end
+    rescue RedisClient::RESP3::UnknownType => error
+      raise RedisClient::ProtocolError, error.message
     rescue SystemCallError, IOError, OpenSSL::SSL::SSLError => error
       raise ConnectionError, error.message
     end

data/lib/redis_client/sentinel_config.rb

@@ -12,8 +12,21 @@ class RedisClient
         raise ArgumentError, "Expected role to be either :master or :replica, got: #{role.inspect}"
       end
 
+      @to_list_of_hash = @to_hash = nil
+      extra_config = {}
+      if client_config[:protocol] == 2
+        extra_config[:protocol] = client_config[:protocol]
+        @to_list_of_hash = lambda do |may_be_a_list|
+          if may_be_a_list.is_a?(Array)
+            may_be_a_list.map { |l| l.each_slice(2).to_h }
+          else
+            may_be_a_list
+          end
+        end
+      end
+
       @name = name
-      @sentinel_configs = sentinels.map { |s| Config.new(**s) }
+      @sentinel_configs = sentinels.map { |s| Config.new(**extra_config, **s) }
       @sentinels = {}.compare_by_identity
       @role = role
       @mutex = Mutex.new
@@ -90,7 +103,10 @@ class RedisClient
           return Config.new(host: host, port: Integer(port), **@client_config)
         end
       end
-      raise ConnectionError, "Couldn't locate a master for role: #{@name}"
+    rescue ConnectionError
+      raise ConnectionError, "No sentinels available"
+    else
+      raise ConnectionError, "Couldn't locate a replica for role: #{@name}"
     end
 
     def sentinel_client(sentinel_config)
@@ -99,13 +115,19 @@ class RedisClient
 
     def resolve_replica
       each_sentinel do |sentinel_client|
-        replicas = sentinel_client.call("SENTINEL", "replicas", @name)
+        replicas = sentinel_client.call("SENTINEL", "replicas", @name, &@to_list_of_hash)
+        replicas.reject! do |r|
+          flags = r["flags"].to_s.split(",")
+          flags.include?("s_down") || flags.include?("o_down")
+        end
         next if replicas.empty?
 
-        replica = replicas.reject { |r| r["flags"].to_s.split(",").include?("disconnected") }.sample
-        replica ||= replicas.sample
+        replica = replicas.sample
         return Config.new(host: replica["ip"], port: Integer(replica["port"]), **@client_config)
       end
+    rescue ConnectionError
+      raise ConnectionError, "No sentinels available"
+    else
       raise ConnectionError, "Couldn't locate a replica for role: #{@name}"
     end
 

data/lib/redis_client/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class RedisClient
-  VERSION = "0.5.1"
+  VERSION = "0.6.2"
 end

data/lib/redis_client.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 require "redis_client/version"
 require "redis_client/command_builder"
 require "redis_client/config"
@@ -78,17 +76,18 @@ class RedisClient
 
   Error = Class.new(StandardError)
 
+  ProtocolError = Class.new(Error)
   UnsupportedServer = Class.new(Error)
 
   ConnectionError = Class.new(Error)
+  CannotConnectError = Class.new(ConnectionError)
 
   FailoverError = Class.new(ConnectionError)
 
   TimeoutError = Class.new(ConnectionError)
   ReadTimeoutError = Class.new(TimeoutError)
   WriteTimeoutError = Class.new(TimeoutError)
-  ConnectTimeoutError = Class.new(TimeoutError)
-  CheckoutTimeoutError = Class.new(ConnectTimeoutError)
+  CheckoutTimeoutError = Class.new(TimeoutError)
 
   class CommandError < Error
     attr_reader :command
@@ -120,11 +119,11 @@ class RedisClient
 
   class << self
     def config(**kwargs)
-      Config.new(**kwargs)
+      Config.new(client_implementation: self, **kwargs)
     end
 
     def sentinel(**kwargs)
-      SentinelConfig.new(**kwargs)
+      SentinelConfig.new(client_implementation: self, **kwargs)
     end
 
     def new(arg = nil, **kwargs)
@@ -184,7 +183,22 @@ class RedisClient
   end
 
   def call(*command, **kwargs)
-    command = @command_builder.generate!(command, kwargs)
+    command = @command_builder.generate(command, kwargs)
+    result = ensure_connected do |connection|
+      Middlewares.call(command, config) do
+        connection.call(command, nil)
+      end
+    end
+
+    if block_given?
+      yield result
+    else
+      result
+    end
+  end
+
+  def call_v(command)
+    command = @command_builder.generate(command)
     result = ensure_connected do |connection|
       Middlewares.call(command, config) do
         connection.call(command, nil)
@@ -199,7 +213,22 @@ class RedisClient
   end
 
   def call_once(*command, **kwargs)
-    command = @command_builder.generate!(command, kwargs)
+    command = @command_builder.generate(command, kwargs)
+    result = ensure_connected(retryable: false) do |connection|
+      Middlewares.call(command, config) do
+        connection.call(command, nil)
+      end
+    end
+
+    if block_given?
+      yield result
+    else
+      result
+    end
+  end
+
+  def call_once_v(command)
+    command = @command_builder.generate(command)
     result = ensure_connected(retryable: false) do |connection|
       Middlewares.call(command, config) do
         connection.call(command, nil)
@@ -214,14 +243,39 @@ class RedisClient
   end
 
   def blocking_call(timeout, *command, **kwargs)
-    command = @command_builder.generate!(command, kwargs)
+    command = @command_builder.generate(command, kwargs)
+    error = nil
     result = ensure_connected do |connection|
       Middlewares.call(command, config) do
         connection.call(command, timeout)
       end
+    rescue ReadTimeoutError => error
+      break
     end
 
-    if block_given?
+    if error
+      raise error
+    elsif block_given?
+      yield result
+    else
+      result
+    end
+  end
+
+  def blocking_call_v(timeout, command)
+    command = @command_builder.generate(command)
+    error = nil
+    result = ensure_connected do |connection|
+      Middlewares.call(command, config) do
+        connection.call(command, timeout)
+      end
+    rescue ReadTimeoutError => error
+      break
+    end
+
+    if error
+      raise error
+    elsif block_given?
       yield result
     else
       result
@@ -233,7 +287,7 @@ class RedisClient
       return to_enum(__callee__, *args, **kwargs)
     end
 
-    args = @command_builder.generate!(["SCAN", 0] + args, kwargs)
+    args = @command_builder.generate(["SCAN", 0] + args, kwargs)
     scan_list(1, args, &block)
   end
 
@@ -242,7 +296,7 @@ class RedisClient
       return to_enum(__callee__, key, *args, **kwargs)
     end
 
-    args = @command_builder.generate!(["SSCAN", key, 0] + args, kwargs)
+    args = @command_builder.generate(["SSCAN", key, 0] + args, kwargs)
     scan_list(2, args, &block)
   end
 
@@ -251,7 +305,7 @@ class RedisClient
       return to_enum(__callee__, key, *args, **kwargs)
     end
 
-    args = @command_builder.generate!(["HSCAN", key, 0] + args, kwargs)
+    args = @command_builder.generate(["HSCAN", key, 0] + args, kwargs)
     scan_pairs(2, args, &block)
   end
 
@@ -260,7 +314,7 @@ class RedisClient
       return to_enum(__callee__, key, *args, **kwargs)
     end
 
-    args = @command_builder.generate!(["ZSCAN", key, 0] + args, kwargs)
+    args = @command_builder.generate(["ZSCAN", key, 0] + args, kwargs)
     scan_pairs(2, args, &block)
   end
 
@@ -343,7 +397,12 @@ class RedisClient
     end
 
     def call(*command, **kwargs)
-      raw_connection.write(@command_builder.generate!(command, kwargs))
+      raw_connection.write(@command_builder.generate(command, kwargs))
+      nil
+    end
+
+    def call_v(command)
+      raw_connection.write(@command_builder.generate(command))
       nil
     end
 
@@ -378,14 +437,29 @@ class RedisClient
     end
 
     def call(*command, **kwargs, &block)
-      command = @command_builder.generate!(command, kwargs)
+      command = @command_builder.generate(command, kwargs)
+      (@blocks ||= [])[@commands.size] = block if block_given?
+      @commands << command
+      nil
+    end
+
+    def call_v(command, &block)
+      command = @command_builder.generate(command)
       (@blocks ||= [])[@commands.size] = block if block_given?
       @commands << command
       nil
     end
 
-    def call_once(*command, **kwargs)
-      command = @command_builder.generate!(command, kwargs)
+    def call_once(*command, **kwargs, &block)
+      command = @command_builder.generate(command, kwargs)
+      @retryable = false
+      (@blocks ||= [])[@commands.size] = block if block_given?
+      @commands << command
+      nil
+    end
+
+    def call_once_v(command, &block)
+      command = @command_builder.generate(command)
       @retryable = false
       (@blocks ||= [])[@commands.size] = block if block_given?
       @commands << command
@@ -417,18 +491,14 @@ class RedisClient
     end
 
     def _coerce!(results)
-      if results
-        results.each_with_index do |result, index|
-          if result.is_a?(CommandError)
-            result._set_command(@commands[index + 1])
-            raise result
-          end
+      results&.each_with_index do |result, index|
+        if result.is_a?(CommandError)
+          result._set_command(@commands[index + 1])
+          raise result
         end
 
-        @blocks&.each_with_index do |block, index|
-          if block
-            results[index - 1] = block.call(results[index - 1])
-          end
+        if @blocks && block = @blocks[index + 1]
+          results[index] = block.call(result)
         end
       end
 
@@ -442,8 +512,17 @@ class RedisClient
       @timeouts = nil
     end
 
-    def blocking_call(timeout, *command, **kwargs)
-      command = @command_builder.generate!(command, kwargs)
+    def blocking_call(timeout, *command, **kwargs, &block)
+      command = @command_builder.generate(command, kwargs)
+      @timeouts ||= []
+      @timeouts[@commands.size] = timeout
+      (@blocks ||= [])[@commands.size] = block if block_given?
+      @commands << command
+      nil
+    end
+
+    def blocking_call_v(timeout, command, &block)
+      command = @command_builder.generate(command)
       @timeouts ||= []
       @timeouts[@commands.size] = timeout
       (@blocks ||= [])[@commands.size] = block if block_given?
@@ -521,7 +600,7 @@ class RedisClient
         else
           connection
         end
-      rescue ConnectionError => error
+      rescue ConnectionError, ProtocolError => error
         connection&.close
         close
 
@@ -538,6 +617,10 @@ class RedisClient
       begin
         @disable_reconnection = true
         yield connection
+      rescue ConnectionError, ProtocolError
+        connection&.close
+        close
+        raise
       ensure
         @disable_reconnection = previous_disable_reconnection
       end
@@ -568,10 +651,16 @@ class RedisClient
       role, = connection.call_pipelined(prelude, nil).last
       config.check_role!(role)
     else
-      connection.call_pipelined(prelude, nil)
+      unless prelude.empty?
+        connection.call_pipelined(prelude, nil)
+      end
     end
 
     connection
+  rescue FailoverError
+    raise
+  rescue ConnectionError => error
+    raise CannotConnectError, error.message, error.backtrace
   rescue CommandError => error
     if error.message.include?("ERR unknown command `HELLO`")
       raise UnsupportedServer,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-client
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-02 00:00:00.000000000 Z
+date: 2022-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool