redis-cluster-client 0.0.7 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aff938c0517542c32f98206c28f1f605a52751e2e2e9a1bd10f5576285f568ee
-  data.tar.gz: 8a3d45fba93334959b1ee67504d2703bb116a8133ef8dac2aa8f314fdb74b268
+  metadata.gz: 86d73675660702eb937b5d27925248d16b64fe46306d49da0c2b89ee49b5dc34
+  data.tar.gz: d2472791161a5ba746cc29c504353dc0e8414c093a531ff1017653a5a354d7dd
 SHA512:
-  metadata.gz: 63dec0155a50a0dbb7a17763c43291d371abe4e3df407340ba8edf125cf2c2bb338f607026a1d004dca489761cb2e6b5450e68a468f18799e5c3443b52a21be7
-  data.tar.gz: 4de6c4688ca120d7f4b636bd884195b3cddc3bf8d217ea662a1f382ced46de2c9e24b0e967a801d175f0572a083b9e5ea835d54797422cf60ce799cda3002f74
+  metadata.gz: bd12c8ebbc013d36ead0ee5bb4f8914ad8e5dec4a302e1aa54890ba36abd2358376bc0329e94b859c9e5c5d71d1a07d11a2f9684db655fef375000e792ca5f0c
+  data.tar.gz: d05782f64503cc104ef789683448d967ca6d71355cdc8a1411e8a92ea98f9412ad94a7a411bc0a93ffffb5ff384f6d3fb6e48ef83cc07bf0fe907872045e2567

data/lib/redis_client/cluster/command.rb

@@ -7,15 +7,21 @@ class RedisClient
   class Cluster
     class Command
       class << self
-        def load(nodes)
-          errors = nodes&.map do |node|
+        def load(nodes) # rubocop:disable Metrics/MethodLength
+          errors = []
+          cmd = nil
+          nodes&.each do |node|
+            break unless cmd.nil?
+
             reply = node.call('COMMAND')
             details = parse_command_details(reply)
-            return ::RedisClient::Cluster::Command.new(details)
-          rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
-            e
+            cmd = ::RedisClient::Cluster::Command.new(details)
+          rescue ::RedisClient::Error => e
+            errors << e
           end
 
+          return cmd unless cmd.nil?
+
           raise ::RedisClient::Cluster::InitialSetupError, errors
         end
 
@@ -68,12 +74,14 @@ class RedisClient
         @details.fetch(name).fetch(key)
       end
 
-      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity
+      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
         case command&.flatten&.first.to_s.downcase
-        when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
+        when 'eval', 'evalsha', 'zinterstore', 'zunionstore' then 3
         when 'object' then 2
         when 'memory'
           command[1].to_s.casecmp('usage').zero? ? 2 : 0
+        when 'migrate'
+          command[3] == '""' ? determine_optional_key_position(command, 'keys') : 3
         when 'xread', 'xreadgroup'
           determine_optional_key_position(command, 'streams')
         else

data/lib/redis_client/cluster/errors.rb

@@ -6,8 +6,6 @@ class RedisClient
   class Cluster
     ERR_ARG_NORMALIZATION = ->(arg) { Array[arg].flatten.reject { |e| e.nil? || (e.respond_to?(:empty?) && e.empty?) } }
 
-    # Raised when client connected to redis as cluster mode
-    # and failed to fetch cluster state information by commands.
     class InitialSetupError < ::RedisClient::Error
       def initialize(errors)
         msg = ERR_ARG_NORMALIZATION.call(errors).map(&:message).uniq.join(',')
@@ -15,8 +13,6 @@ class RedisClient
       end
     end
 
-    # Raised when client connected to redis as cluster mode
-    # and some cluster subcommands were called.
     class OrchestrationCommandNotSupported < ::RedisClient::Error
       def initialize(command)
         str = ERR_ARG_NORMALIZATION.call(command).map(&:to_s).join(' ').upcase
@@ -28,7 +24,6 @@ class RedisClient
       end
     end
 
-    # Raised when error occurs on any node of cluster.
     class ErrorCollection < ::RedisClient::Error
       attr_reader :errors
 
@@ -41,15 +36,24 @@ class RedisClient
 
         @errors = errors
         messages = @errors.map { |node_key, error| "#{node_key}: #{error.message}" }
-        super("Command errors were replied on any node: #{messages.join(', ')}")
+        super("Errors occurred on any node: #{messages.join(', ')}")
       end
     end
 
-    # Raised when cluster client can't select node.
     class AmbiguousNodeError < ::RedisClient::Error
       def initialize(command)
         super("Cluster client doesn't know which node the #{command} command should be sent to.")
       end
     end
+
+    class NodeMightBeDown < ::RedisClient::Error
+      def initialize(_ = '')
+        super(
+          'The client is trying to fetch the latest cluster state '\
+          'because a subset of nodes might be down. '\
+          'It might continue to raise errors for a while.'
+        )
+      end
+    end
   end
 end

data/lib/redis_client/cluster/key_slot_converter.rb

@@ -43,6 +43,8 @@ class RedisClient
       module_function
 
       def convert(key)
+        return nil if key.nil?
+
         crc = 0
         key.each_byte do |b|
           crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]

data/lib/redis_client/cluster/node.rb

@@ -12,6 +12,7 @@ class RedisClient
       SLOT_SIZE = 16_384
       MIN_SLOT = 0
       MAX_SLOT = SLOT_SIZE - 1
+      MAX_STARTUP_SAMPLE = 37
       IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
 
       ReloadNeeded = Class.new(::RedisClient::Error)
@@ -32,19 +33,33 @@ class RedisClient
       end
 
       class << self
-        def load_info(options, **kwargs)
-          startup_nodes = ::RedisClient::Cluster::Node.new(options, **kwargs)
-
-          errors = startup_nodes.map do |n|
-            reply = n.call('CLUSTER', 'NODES')
-            return parse_node_info(reply)
-          rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
-            e
+        def load_info(options, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+          startup_size = options.size > MAX_STARTUP_SAMPLE ? MAX_STARTUP_SAMPLE : options.size
+          node_info_list = Array.new(startup_size)
+          errors = Array.new(startup_size)
+          startup_options = options.to_a.sample(MAX_STARTUP_SAMPLE).to_h
+          startup_nodes = ::RedisClient::Cluster::Node.new(startup_options, **kwargs)
+          threads = startup_nodes.each_with_index.map do |raw_client, idx|
+            Thread.new(raw_client, idx) do |cli, i|
+              Thread.pass
+              reply = cli.call('CLUSTER', 'NODES')
+              node_info_list[i] = parse_node_info(reply)
+            rescue StandardError => e
+              errors[i] = e
+            ensure
+              cli&.close
+            end
+          end
+          threads.each(&:join)
+          raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.all?(&:nil?)
+
+          grouped = node_info_list.compact.group_by do |rows|
+            rows.sort_by { |row| row[:id] }
+                .map { |r| "#{r[:id]}#{r[:node_key]}#{r[:role]}#{r[:primary_id]}#{r[:config_epoch]}" }
+                .join
           end
 
-          raise ::RedisClient::Cluster::InitialSetupError, errors
-        ensure
-          startup_nodes&.each(&:close)
+          grouped.max_by { |_, v| v.size }[1].first
         end
 
         private
@@ -87,7 +102,7 @@ class RedisClient
       end
 
       def each(&block)
-        @clients.values.each(&block)
+        @clients.each_value(&block)
       end
 
       def sample
@@ -109,32 +124,58 @@ class RedisClient
       end
 
       def find_by(node_key)
+        raise ReloadNeeded if node_key.nil? || !@clients.key?(node_key)
+
         @clients.fetch(node_key)
-      rescue KeyError
-        raise ReloadNeeded
       end
 
-      def call_all(method, *command, **kwargs, &block)
-        try_map { |_, client| client.send(method, *command, **kwargs, &block) }.values
+      def call_all(method, *args, **kwargs, &block)
+        results, errors = try_map do |_, client|
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_primaries(method, *command, **kwargs, &block)
-        try_map do |node_key, client|
+      def call_primaries(method, *args, **kwargs, &block)
+        results, errors = try_map do |node_key, client|
           next if replica?(node_key)
 
-          client.send(method, *command, **kwargs, &block)
-        end.values
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_replicas(method, *command, **kwargs, &block)
-        return call_primaries(method, *command, **kwargs, &block) if replica_disabled?
+      def call_replicas(method, *args, **kwargs, &block)
+        return call_primaries(method, *args, **kwargs, &block) if replica_disabled?
 
         replica_node_keys = @replications.values.map(&:sample)
-        try_map do |node_key, client|
+        results, errors = try_map do |node_key, client|
           next if primary?(node_key) || !replica_node_keys.include?(node_key)
 
-          client.send(method, *command, **kwargs, &block)
-        end.values
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
+      end
+
+      def send_ping(method, *args, **kwargs, &block)
+        results, errors = try_map do |_, client|
+          client.send(method, *args, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ReloadNeeded if errors.values.any?(::RedisClient::ConnectionError)
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
       def scale_reading_clients
@@ -144,14 +185,9 @@ class RedisClient
         end
       end
 
-      def slot_exists?(slot)
-        slot = Integer(slot)
-        return false if slot < MIN_SLOT || slot > MAX_SLOT
-
-        !@slots[slot].nil?
-      end
-
       def find_node_key_of_primary(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -159,6 +195,8 @@ class RedisClient
       end
 
       def find_node_key_of_replica(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -241,9 +279,7 @@ class RedisClient
         end
 
         threads.each(&:join)
-        return results if errors.empty?
-
-        raise ::RedisClient::Cluster::ErrorCollection, errors
+        [results, errors]
       end
     end
   end

data/lib/redis_client/cluster.rb

@@ -127,8 +127,7 @@ class RedisClient
     end
 
     def blocking_call(timeout, *command, **kwargs)
-      node = assign_node(*command)
-      try_send(node, :blocking_call, timeout, *command, **kwargs)
+      send_command(:blocking_call, timeout, *command, **kwargs)
     end
 
     def scan(*args, **kwargs, &block)
@@ -170,8 +169,10 @@ class RedisClient
     end
 
     def close
-      @node.each(&:close)
+      @node.call_all(:close)
       nil
+    rescue StandardError
+      # ignore
     end
 
     private
@@ -184,106 +185,124 @@ class RedisClient
                                        node_info: node_info, pool: pool, with_replica: config.use_replica?, **kwargs)
     end
 
-    def send_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    def send_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       cmd = command.first.to_s.downcase
       case cmd
-      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save', 'ping'
-        @node.call_all(method, *command, **kwargs, &block).first
+      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
+        @node.call_all(method, *args, **kwargs, &block).first
       when 'flushall', 'flushdb'
-        @node.call_primaries(method, *command, **kwargs, &block).first
-      when 'wait'     then send_wait_command(method, *command, **kwargs, &block)
-      when 'keys'     then @node.call_replicas(method, *command, **kwargs, &block).flatten.sort
-      when 'dbsize'   then @node.call_replicas(method, *command, **kwargs, &block).sum
+        @node.call_primaries(method, *args, **kwargs, &block).first
+      when 'ping'     then @node.send_ping(method, *args, **kwargs, &block).first
+      when 'wait'     then send_wait_command(method, *args, **kwargs, &block)
+      when 'keys'     then @node.call_replicas(method, *args, **kwargs, &block).flatten.sort
+      when 'dbsize'   then @node.call_replicas(method, *args, **kwargs, &block).sum
       when 'scan'     then _scan(*command, **kwargs)
-      when 'lastsave' then @node.call_all(method, *command, **kwargs, &block).sort
-      when 'role'     then @node.call_all(method, *command, **kwargs, &block)
-      when 'config'   then send_config_command(method, *command, **kwargs, &block)
-      when 'client'   then send_client_command(method, *command, **kwargs, &block)
-      when 'cluster'  then send_cluster_command(method, *command, **kwargs, &block)
+      when 'lastsave' then @node.call_all(method, *args, **kwargs, &block).sort
+      when 'role'     then @node.call_all(method, *args, **kwargs, &block)
+      when 'config'   then send_config_command(method, *args, **kwargs, &block)
+      when 'client'   then send_client_command(method, *args, **kwargs, &block)
+      when 'cluster'  then send_cluster_command(method, *args, **kwargs, &block)
       when 'readonly', 'readwrite', 'shutdown'
         raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, cmd
-      when 'memory'   then send_memory_command(method, *command, **kwargs, &block)
-      when 'script'   then send_script_command(method, *command, **kwargs, &block)
-      when 'pubsub'   then send_pubsub_command(method, *command, **kwargs, &block)
+      when 'memory'   then send_memory_command(method, *args, **kwargs, &block)
+      when 'script'   then send_script_command(method, *args, **kwargs, &block)
+      when 'pubsub'   then send_pubsub_command(method, *args, **kwargs, &block)
       when 'discard', 'exec', 'multi', 'unwatch'
         raise ::RedisClient::Cluster::AmbiguousNodeError, cmd
       else
         node = assign_node(*command)
-        try_send(node, method, *command, **kwargs, &block)
+        try_send(node, method, *args, **kwargs, &block)
       end
-    rescue RedisClient::Cluster::ErrorCollection => e
-      update_cluster_info! if e.errors.values.map(&:class).any?(::RedisClient::ConnectionError)
-      raise
+    rescue RedisClient::Cluster::Node::ReloadNeeded
+      update_cluster_info!
+      raise ::RedisClient::Cluster::NodeMightBeDown
     end
 
-    def send_wait_command(method, *command, retry_count: 3, **kwargs, &block)
-      @node.call_primaries(method, *command, **kwargs, &block).sum
+    def send_wait_command(method, *args, retry_count: 3, **kwargs, &block)
+      @node.call_primaries(method, *args, **kwargs, &block).sum
     rescue RedisClient::Cluster::ErrorCollection => e
-      raise if retry_count <= 0 || e.errors.values.map(&:message).grep(/ERR WAIT cannot be used with replica instances/).empty?
+      raise if retry_count <= 0
+      raise if e.errors.values.none? do |err|
+        err.message.include?('WAIT cannot be used with replica instances')
+      end
 
       update_cluster_info!
       retry_count -= 1
       retry
     end
 
-    def send_config_command(method, *command, **kwargs, &block)
+    def send_config_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
       when 'resetstat', 'rewrite', 'set'
-        @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_memory_command(method, *command, **kwargs, &block)
+    def send_memory_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'stats' then @node.call_all(method, *command, **kwargs, &block)
-      when 'purge' then @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+      when 'stats' then @node.call_all(method, *args, **kwargs, &block)
+      when 'purge' then @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_client_command(method, *command, **kwargs, &block)
+    def send_client_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'list' then @node.call_all(method, *command, **kwargs, &block).flatten
+      when 'list' then @node.call_all(method, *args, **kwargs, &block).flatten
       when 'pause', 'reply', 'setname'
-        @node.call_all(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_all(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_cluster_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/MethodLength
+    def send_cluster_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
       subcommand = command[1].to_s.downcase
+
       case subcommand
       when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
            'reset', 'set-config-epoch', 'setslot'
         raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, ['cluster', subcommand]
-      when 'saveconfig' then @node.call_all(method, *command, **kwargs, &block).first
+      when 'saveconfig' then @node.call_all(method, *args, **kwargs, &block).first
       when 'getkeysinslot'
         raise ArgumentError, command.join(' ') if command.size != 4
 
-        find_node(@node.find_node_key_of_replica(command[2])).send(method, *command, **kwargs, &block)
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        find_node(@node.find_node_key_of_replica(command[2])).send(method, *args, **kwargs, &block)
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_script_command(method, *command, **kwargs, &block)
+    def send_script_command(method, *args, **kwargs, &block)
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
       when 'debug', 'kill'
-        @node.call_all(method, *command, **kwargs, &block).first
+        @node.call_all(method, *args, **kwargs, &block).first
       when 'flush', 'load'
-        @node.call_primaries(method, *command, **kwargs, &block).first
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+        @node.call_primaries(method, *args, **kwargs, &block).first
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
-    def send_pubsub_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+    def send_pubsub_command(method, *args, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      command = method == :blocking_call && args.size > 1 ? args[1..] : args
+
       case command[1].to_s.downcase
-      when 'channels' then @node.call_all(method, *command, **kwargs, &block).flatten.uniq.sort
+      when 'channels' then @node.call_all(method, *args, **kwargs, &block).flatten.uniq.sort
       when 'numsub'
-        @node.call_all(method, *command, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
+        @node.call_all(method, *args, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
              .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
-      when 'numpat' then @node.call_all(method, *command, **kwargs, &block).sum
-      else assign_node(*command).send(method, *command, **kwargs, &block)
+      when 'numpat' then @node.call_all(method, *args, **kwargs, &block).sum
+      else assign_node(*command).send(method, *args, **kwargs, &block)
       end
     end
 
@@ -353,44 +372,30 @@ class RedisClient
       find_node(node_key)
     end
 
-    def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
+    def find_node_key(*command, primary_only: false)
       key = @command.extract_first_key(command)
-      if key.empty?
-        return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
-
-        return @node.replica_node_keys.sample
-      end
-
-      slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
-      return unless @node.slot_exists?(slot)
+      slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
 
       if @command.should_send_to_primary?(command) || primary_only
-        @node.find_node_key_of_primary(slot)
+        @node.find_node_key_of_primary(slot) || @node.primary_node_keys.sample
       else
-        @node.find_node_key_of_replica(slot)
+        @node.find_node_key_of_replica(slot) || @node.replica_node_keys.sample
       end
     end
 
     def find_node(node_key, retry_count: 3)
-      return @node.sample if node_key.nil?
-
       @node.find_by(node_key)
     rescue ::RedisClient::Cluster::Node::ReloadNeeded
-      raise(::RedisClient::ConnectionError, 'unstable cluster state') if retry_count <= 0
+      raise ::RedieClient::Cluster::NodeMightBeDown if retry_count <= 0
 
-      update_cluster_info!(node_key)
+      update_cluster_info!
       retry_count -= 1
       retry
     end
 
-    def update_cluster_info!(node_key = nil)
+    def update_cluster_info!
       @mutex.synchronize do
-        unless node_key.nil?
-          host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
-          @config.add_node(host, port)
-        end
-
-        @node.each(&:close)
+        close
         @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.10
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-19 00:00:00.000000000 Z
+date: 2022-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client