redis-cluster-client 0.0.6 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fde84481c2ca9680670f252a002391a34ae793148a21a8e02086d199412a53ab
-  data.tar.gz: 5af188432f9ae5e76c13d0dd174c25cde4b98d1de49efeb97e2a809bd111a9a6
+  metadata.gz: 032431ff16773725b9cb68482bc5df78affdf23e66b6bbd91b6ddbabdd9da921
+  data.tar.gz: dc789f59f6509e35ab23df03a70867b5e8414ee3bb327462fbcd5d63be4c6744
 SHA512:
-  metadata.gz: f26dc7bb447ee70742efc90344758ad3058cd0d1506ad8576637b7e9ad296a806d7e15e3b824cead0fb2313140dea00f6f579a15803399c16257ac347c45085f
-  data.tar.gz: 06f43e223a53d9ada0803f7292bc1ca58505fb3c341fd7643eb6cf0eff73bebe4ecd2022b0a1b59af81dda879a9d8a9b078107181ff45037773382dada321597
+  metadata.gz: 6013ff65d8dc06b776b7b201244d75eaf62993368f5285d823472f78bec2c81639734f23c32793bc1b8d1981905e5794e35e4e353696d70f1d66930434e6104f
+  data.tar.gz: 0e1dc5f17a9706c2fc64bdd03e74ee457f9b9aea81740cdeb5459bd9bbc769b6353047b64ef008b6d34b950fbc7cb8cf57e66c9deb41d33edbacaba78ca2a12e

data/lib/redis_client/cluster/command.rb

@@ -68,12 +68,14 @@ class RedisClient
         @details.fetch(name).fetch(key)
       end
 
-      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity
+      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
         case command&.flatten&.first.to_s.downcase
-        when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
+        when 'eval', 'evalsha', 'zinterstore', 'zunionstore' then 3
         when 'object' then 2
         when 'memory'
           command[1].to_s.casecmp('usage').zero? ? 2 : 0
+        when 'migrate'
+          command[3] == '""' ? determine_optional_key_position(command, 'keys') : 3
         when 'xread', 'xreadgroup'
           determine_optional_key_position(command, 'streams')
         else

data/lib/redis_client/cluster/errors.rb

@@ -29,7 +29,7 @@ class RedisClient
     end
 
     # Raised when error occurs on any node of cluster.
-    class CommandErrorCollection < ::RedisClient::Error
+    class ErrorCollection < ::RedisClient::Error
       attr_reader :errors
 
       def initialize(errors)
@@ -51,5 +51,15 @@ class RedisClient
         super("Cluster client doesn't know which node the #{command} command should be sent to.")
       end
     end
+
+    class NodeMightBeDown < ::RedisClient::Error
+      def initialize(_ = '')
+        super(
+          'The client is trying to fetch the latest cluster state '\
+          'because a subset of nodes might be down. '\
+          'It might continue to raise errors for a while.'
+        )
+      end
+    end
   end
 end

data/lib/redis_client/cluster/key_slot_converter.rb

@@ -43,6 +43,8 @@ class RedisClient
       module_function
 
       def convert(key)
+        return nil if key.nil?
+
         crc = 0
         key.each_byte do |b|
           crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]

data/lib/redis_client/cluster/node.rb

@@ -12,6 +12,7 @@ class RedisClient
       SLOT_SIZE = 16_384
       MIN_SLOT = 0
       MAX_SLOT = SLOT_SIZE - 1
+      MAX_STARTUP_SAMPLE = 37
       IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
 
       ReloadNeeded = Class.new(::RedisClient::Error)
@@ -32,19 +33,33 @@ class RedisClient
       end
 
       class << self
-        def load_info(options, **kwargs)
-          startup_nodes = ::RedisClient::Cluster::Node.new(options, **kwargs)
-
-          errors = startup_nodes.map do |n|
-            reply = n.call('CLUSTER', 'NODES')
-            return parse_node_info(reply)
-          rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
-            e
+        def load_info(options, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+          startup_size = options.size > MAX_STARTUP_SAMPLE ? MAX_STARTUP_SAMPLE : options.size
+          node_info_list = Array.new(startup_size)
+          errors = Array.new(startup_size)
+          startup_options = options.to_a.sample(MAX_STARTUP_SAMPLE).to_h
+          startup_nodes = ::RedisClient::Cluster::Node.new(startup_options, **kwargs)
+          threads = startup_nodes.each_with_index.map do |raw_client, idx|
+            Thread.new(raw_client, idx) do |cli, i|
+              Thread.pass
+              reply = cli.call('CLUSTER', 'NODES')
+              node_info_list[i] = parse_node_info(reply)
+            rescue StandardError => e
+              errors[i] = e
+            ensure
+              cli&.close
+            end
+          end
+          threads.each(&:join)
+          raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.all?(&:nil?)
+
+          grouped = node_info_list.compact.group_by do |rows|
+            rows.sort_by { |row| row[:id] }
+                .map { |r| "#{r[:id]}#{r[:node_key]}#{r[:role]}#{r[:primary_id]}#{r[:config_epoch]}" }
+                .join
           end
 
-          raise ::RedisClient::Cluster::InitialSetupError, errors
-        ensure
-          startup_nodes&.each(&:close)
+          grouped.max_by { |_, v| v.size }[1].first
         end
 
         private
@@ -109,32 +124,58 @@ class RedisClient
       end
 
       def find_by(node_key)
+        raise ReloadNeeded if node_key.nil? || !@clients.key?(node_key)
+
         @clients.fetch(node_key)
-      rescue KeyError
-        raise ReloadNeeded
       end
 
-      def call_all(method, *command, **kwargs, &block)
-        try_map { |_, client| client.send(method, *command, **kwargs, &block) }.values
+      def call_all(method, *args, **kwargs, &block)
+        results, errors = try_map do |_, client|
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_primary(method, *command, **kwargs, &block)
-        try_map do |node_key, client|
+      def call_primaries(method, *args, **kwargs, &block)
+        results, errors = try_map do |node_key, client|
           next if replica?(node_key)
 
-          client.send(method, *command, **kwargs, &block)
-        end.values
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_replica(method, *command, **kwargs, &block)
-        return call_primary(method, *command, **kwargs, &block) if replica_disabled?
+      def call_replicas(method, *args, **kwargs, &block)
+        return call_primaries(method, *args, **kwargs, &block) if replica_disabled?
 
         replica_node_keys = @replications.values.map(&:sample)
-        try_map do |node_key, client|
+        results, errors = try_map do |node_key, client|
           next if primary?(node_key) || !replica_node_keys.include?(node_key)
 
-          client.send(method, *command, **kwargs, &block)
-        end.values
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
+      end
+
+      def send_ping(method, *args, **kwargs, &block)
+        results, errors = try_map do |_, client|
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ReloadNeeded if errors.values.any?(::RedisClient::ConnectionError)
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
       def scale_reading_clients
@@ -144,14 +185,9 @@ class RedisClient
         end
       end
 
-      def slot_exists?(slot)
-        slot = Integer(slot)
-        return false if slot < MIN_SLOT || slot > MAX_SLOT
-
-        !@slots[slot].nil?
-      end
-
       def find_node_key_of_primary(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -159,6 +195,8 @@ class RedisClient
       end
 
       def find_node_key_of_replica(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -228,22 +266,20 @@ class RedisClient
       end
 
       def try_map # rubocop:disable Metrics/MethodLength
-        errors = {}
         results = {}
+        errors = {}
         threads = @clients.map do |k, v|
           Thread.new(k, v) do |node_key, client|
             Thread.pass
             reply = yield(node_key, client)
             results[node_key] = reply unless reply.nil?
-          rescue ::RedisClient::CommandError => e
+          rescue StandardError => e
             errors[node_key] = e
           end
         end
 
         threads.each(&:join)
-        return results if errors.empty?
-
-        raise ::RedisClient::Cluster::CommandErrorCollection, errors
+        [results, errors]
       end
     end
   end

data/lib/redis_client/cluster.rb

@@ -40,9 +40,10 @@ class RedisClient
         @size.zero?
       end
 
-      # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37
-      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37 handle redirections
+      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         all_replies = Array.new(@size)
+        errors = {}
         threads = @grouped.map do |k, v|
           Thread.new(@client, k, v) do |client, node_key, rows|
             Thread.pass
@@ -60,11 +61,15 @@ class RedisClient
             raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
 
             rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
+          rescue StandardError => e
+            errors[node_key] = e
           end
         end
 
         threads.each(&:join)
-        all_replies
+        return all_replies if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
     end
 
@@ -122,8 +127,7 @@ class RedisClient
     end
 
     def blocking_call(timeout, *command, **kwargs)
-      node = assign_node(*command)
-      try_send(node, :blocking_call, timeout, *command, **kwargs)
+      send_command(:blocking_call, timeout, *command, **kwargs)
     end
 
     def scan(*args, **kwargs, &block)
@@ -165,8 +169,10 @@ class RedisClient
     end
 
     def close
-      @node.each(&:close)
+      @node.call_all(:close)
       nil
+    rescue StandardError
+      # ignore
     end
 
     private
@@ -179,102 +185,124 @@ class RedisClient
                                        node_info: node_info, pool: pool, with_replica: config.use_replica?, **kwargs)
     end
 
-    def send_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    def send_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       cmd = command.first.to_s.downcase
       case cmd
-      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save', 'ping'
-        @node.call_all(method, *command, **kwargs, &block).first
+      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
+        @node.call_all(method, *args, **kwargs, &block).first
       when 'flushall', 'flushdb'
-        @node.call_primary(method, *command, **kwargs, &block).first
-      when 'wait'     then send_wait_command(method, *command, **kwargs, &block)
-      when 'keys'     then @node.call_replica(method, *command, **kwargs, &block).flatten.sort
-      when 'dbsize'   then @node.call_replica(method, *command, **kwargs, &block).sum
+        @node.call_primaries(method, *args, **kwargs, &block).first
+      when 'ping'     then @node.send_ping(method, *args, **kwargs, &block).first
+      when 'wait'     then send_wait_command(method, *args, **kwargs, &block)
+      when 'keys'     then @node.call_replicas(method, *args, **kwargs, &block).flatten.sort
+      when 'dbsize'   then @node.call_replicas(method, *args, **kwargs, &block).sum
       when 'scan'     then _scan(*command, **kwargs)
-      when 'lastsave' then @node.call_all(method, *command, **kwargs, &block).sort
-      when 'role'     then @node.call_all(method, *command, **kwargs, &block)
-      when 'config'   then send_config_command(method, *command, **kwargs, &block)
-      when 'client'   then send_client_command(method, *command, **kwargs, &block)
-      when 'cluster'  then send_cluster_command(method, *command, **kwargs, &block)
+      when 'lastsave' then @node.call_all(method, *args, **kwargs, &block).sort
+      when 'role'     then @node.call_all(method, *args, **kwargs, &block)
+      when 'config'   then send_config_command(method, *args, **kwargs, &block)
+      when 'client'   then send_client_command(method, *args, **kwargs, &block)
+      when 'cluster'  then send_cluster_command(method, *args, **kwargs, &block)
       when 'readonly', 'readwrite', 'shutdown'
         raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, cmd
-      when 'memory'   then send_memory_command(method, *command, **kwargs, &block)
-      when 'script'   then send_script_command(method, *command, **kwargs, &block)
-      when 'pubsub'   then send_pubsub_command(method, *command, **kwargs, &block)
+      when 'memory'   then send_memory_command(method, *args, **kwargs, &block)
+      when 'script'   then send_script_command(method, *args, **kwargs, &block)
+      when 'pubsub'   then send_pubsub_command(method, *args, **kwargs, &block)
       when 'discard', 'exec', 'multi', 'unwatch'
         raise ::RedisClient::Cluster::AmbiguousNodeError, cmd
       else
         node = assign_node(*command)
-        try_send(node, method, *command, **kwargs, &block)
+        try_send(node, method, *args, **kwargs, &block)
       end
-    rescue RedisClient::Cluster::CommandErrorCollection => e
-      update_cluster_info! if e.errors.values.map(&:class).any?(::RedisClient::ConnectionError)
-      raise
+    rescue RedisClient::Cluster::Node::ReloadNeeded
+      update_cluster_info!
+      raise ::RedisClient::Cluster::NodeMightBeDown
     end
 
-    def send_wait_command(method, *command, retry_count: 3, **kwargs, &block)
-      @node.call_primary(method, *command, **kwargs, &block).sum
-    rescue RedisClient::Cluster::CommandErrorCollection => e
-      raise if retry_count <= 0 || e.errors.values.map(&:message).grep(/ERR WAIT cannot be used with replica instances/).empty?
+    def send_wait_command(method, *args, retry_count: 3, **kwargs, &block)
+      @node.call_primaries(method, *args, **kwargs, &block).sum
+    rescue RedisClient::Cluster::ErrorCollection => e
+      raise if retry_count <= 0
+      raise if e.errors.values.none? do |err|
+        err.message.include?('WAIT cannot be used with replica instances')
+      end
 
       update_cluster_info!
       retry_count -= 1
       retry
     end
 
-    def send_config_command(method, *command, **kwargs, &block)
+    def send_config_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
       when 'resetstat', 'rewrite', 'set'
-        @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_memory_command(method, *command, **kwargs, &block)
+    def send_memory_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'stats' then @node.call_all(method, *command, **kwargs, &block)
-      when 'purge' then @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+      when 'stats' then @node.call_all(method, *args, **kwargs, &block)
+      when 'purge' then @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_client_command(method, *command, **kwargs, &block)
+    def send_client_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'list' then @node.call_all(method, *command, **kwargs, &block).flatten
+      when 'list' then @node.call_all(method, *args, **kwargs, &block).flatten
       when 'pause', 'reply', 'setname'
-        @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_cluster_command(method, *command, **kwargs, &block)
+    def send_cluster_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
       subcommand = command[1].to_s.downcase
+
       case subcommand
       when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
            'reset', 'set-config-epoch', 'setslot'
         raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, ['cluster', subcommand]
-      when 'saveconfig' then @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+      when 'saveconfig' then @node.call_all(method, *args, **kwargs, &block).first
+      when 'getkeysinslot'
+        raise ArgumentError, command.join(' ') if command.size != 4
+
+        find_node(@node.find_node_key_of_replica(command[2])).send(method, *args, **kwargs, &block)
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_script_command(method, *command, **kwargs, &block)
+    def send_script_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
       when 'debug', 'kill'
-        @node.call_all(method, *command, **kwargs, &block).first
+        @node.call_all(method, *args, **kwargs, &block).first
       when 'flush', 'load'
-        @node.call_primary(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_primaries(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_pubsub_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+    def send_pubsub_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'channels' then @node.call_all(method, *command, **kwargs, &block).flatten.uniq.sort
+      when 'channels' then @node.call_all(method, *args, **kwargs, &block).flatten.uniq.sort
       when 'numsub'
-        @node.call_all(method, *command, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
+        @node.call_all(method, *args, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
              .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
-      when 'numpat' then @node.call_all(method, *command, **kwargs, &block).sum
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+      when 'numpat' then @node.call_all(method, *args, **kwargs, &block).sum
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
@@ -344,44 +372,30 @@ class RedisClient
       find_node(node_key)
     end
 
-    def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
+    def find_node_key(*command, primary_only: false)
       key = @command.extract_first_key(command)
-      if key.empty?
-        return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
-
-        return @node.replica_node_keys.sample
-      end
-
-      slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
-      return unless @node.slot_exists?(slot)
+      slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
 
       if @command.should_send_to_primary?(command) || primary_only
-        @node.find_node_key_of_primary(slot)
+        @node.find_node_key_of_primary(slot) || @node.primary_node_keys.sample
       else
-        @node.find_node_key_of_replica(slot)
+        @node.find_node_key_of_replica(slot) || @node.replica_node_keys.sample
       end
     end
 
     def find_node(node_key, retry_count: 3)
-      return @node.sample if node_key.nil?
-
       @node.find_by(node_key)
     rescue ::RedisClient::Cluster::Node::ReloadNeeded
-      raise(::RedisClient::ConnectionError, 'unstable cluster state') if retry_count <= 0
+      raise ::RedieClient::Cluster::NodeMightBeDown if retry_count <= 0
 
-      update_cluster_info!(node_key)
+      update_cluster_info!
       retry_count -= 1
       retry
     end
 
-    def update_cluster_info!(node_key = nil)
+    def update_cluster_info!
       @mutex.synchronize do
-        unless node_key.nil?
-          host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
-          @config.add_node(host, port)
-        end
-
-        @node.each(&:close)
+        close
         @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.9
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-19 00:00:00.000000000 Z
+date: 2022-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client