redis-cluster-client 0.0.5 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fce5f4a15cfcab3d7cca37052e9f85ad45abcee8f4de9e458fc560a01382842
-  data.tar.gz: 2da9fb7a3ae8836dd7606fbfd05e33f423529716493d3bb8fccd6d01bae10bc3
+  metadata.gz: d13300ba8c7f294d3e4c86932d13775ad22060c172900c01d521b1ed8c2278d6
+  data.tar.gz: d5b457978bb312a718ba57b48b56fac1ceb07a463889835c8190794007ecbf7d
 SHA512:
-  metadata.gz: fd4b929261fe0a50f4b13fe4963550e7b8c1197ee4f813a464b309140b29d7cc75a95259f100481d2f7e5a958a98608aca980e8308dca125200c1acb4e506e06
-  data.tar.gz: b41237f574ca12b3826e8d869f3ea3251a0acb41961fb009efe7bb5d2f07517b17cae80c5161da91603b4586a63beef11ba618b6386cfd684012cc7fa09752d1
+  metadata.gz: 37fb06428eed5a9cca5b42d27c034f057dd7d6f0c5d0c17e4b42c60a8fa3711127c5ed81f06e0565331f64aa762ccc30ee6674fb541e56a8b322ee50ab7a75ce
+  data.tar.gz: 06d46f93c8c7b7116378d2ef9e4380ccce3aac74c071b709cbacd677ec0b302302b52160c04b2c3523771f69446b96d84e8b4f729765073386c5e5f1f3f27f5b

data/lib/redis_client/cluster/command.rb

@@ -68,12 +68,14 @@ class RedisClient
         @details.fetch(name).fetch(key)
       end
 
-      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity
+      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
         case command&.flatten&.first.to_s.downcase
-        when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
+        when 'eval', 'evalsha', 'zinterstore', 'zunionstore' then 3
         when 'object' then 2
         when 'memory'
           command[1].to_s.casecmp('usage').zero? ? 2 : 0
+        when 'migrate'
+          command[3] == '""' ? determine_optional_key_position(command, 'keys') : 3
         when 'xread', 'xreadgroup'
           determine_optional_key_position(command, 'streams')
         else

data/lib/redis_client/cluster/errors.rb

@@ -29,7 +29,7 @@ class RedisClient
     end
 
     # Raised when error occurs on any node of cluster.
-    class CommandErrorCollection < ::RedisClient::Error
+    class ErrorCollection < ::RedisClient::Error
       attr_reader :errors
 
       def initialize(errors)
@@ -51,5 +51,15 @@ class RedisClient
         super("Cluster client doesn't know which node the #{command} command should be sent to.")
       end
     end
+
+    class NodeMightBeDown < ::RedisClient::Error
+      def initialize(_ = '')
+        super(
+          'The client is trying to fetch the latest cluster state '\
+          'because a subset of nodes might be down. '\
+          'It might continue to raise errors for a while.'
+        )
+      end
+    end
   end
 end

data/lib/redis_client/cluster/key_slot_converter.rb

@@ -43,6 +43,8 @@ class RedisClient
       module_function
 
       def convert(key)
+        return nil if key.nil?
+
         crc = 0
         key.each_byte do |b|
           crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]

data/lib/redis_client/cluster/node.rb

@@ -12,6 +12,7 @@ class RedisClient
       SLOT_SIZE = 16_384
       MIN_SLOT = 0
       MAX_SLOT = SLOT_SIZE - 1
+      MAX_STARTUP_SAMPLE = 37
       IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
 
       ReloadNeeded = Class.new(::RedisClient::Error)
@@ -32,19 +33,33 @@ class RedisClient
       end
 
       class << self
-        def load_info(options, **kwargs)
-          startup_nodes = ::RedisClient::Cluster::Node.new(options, **kwargs)
-
-          errors = startup_nodes.map do |n|
-            reply = n.call('CLUSTER', 'NODES')
-            return parse_node_info(reply)
-          rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
-            e
+        def load_info(options, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+          startup_size = options.size > MAX_STARTUP_SAMPLE ? MAX_STARTUP_SAMPLE : options.size
+          node_info_list = Array.new(startup_size)
+          errors = Array.new(startup_size)
+          startup_options = options.to_a.sample(MAX_STARTUP_SAMPLE).to_h
+          startup_nodes = ::RedisClient::Cluster::Node.new(startup_options, **kwargs)
+          threads = startup_nodes.each_with_index.map do |raw_client, idx|
+            Thread.new(raw_client, idx) do |cli, i|
+              Thread.pass
+              reply = cli.call('CLUSTER', 'NODES')
+              node_info_list[i] = parse_node_info(reply)
+            rescue StandardError => e
+              errors[i] = e
+            ensure
+              cli&.close
+            end
+          end
+          threads.each(&:join)
+          raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.all?(&:nil?)
+
+          grouped = node_info_list.compact.group_by do |rows|
+            rows.sort_by { |row| row[:id] }
+                .map { |r| "#{r[:id]}#{r[:node_key]}#{r[:role]}#{r[:primary_id]}#{r[:config_epoch]}" }
+                .join
           end
 
-          raise ::RedisClient::Cluster::InitialSetupError, errors
-        ensure
-          startup_nodes&.each(&:close)
+          grouped.max_by { |_, v| v.size }[1].first
         end
 
         private
@@ -109,32 +124,58 @@ class RedisClient
       end
 
       def find_by(node_key)
+        raise ReloadNeeded if node_key.nil? || !@clients.key?(node_key)
+
         @clients.fetch(node_key)
-      rescue KeyError
-        raise ReloadNeeded
       end
 
       def call_all(method, *command, **kwargs, &block)
-        try_map { |_, client| client.send(method, *command, **kwargs, &block) }.values
+        results, errors = try_map do |_, client|
+          client.send(method, *command, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_primary(method, *command, **kwargs, &block)
-        try_map do |node_key, client|
+      def call_primaries(method, *command, **kwargs, &block)
+        results, errors = try_map do |node_key, client|
           next if replica?(node_key)
 
           client.send(method, *command, **kwargs, &block)
-        end.values
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def call_replica(method, *command, **kwargs, &block)
-        return call_primary(method, *command, **kwargs, &block) if replica_disabled?
+      def call_replicas(method, *command, **kwargs, &block)
+        return call_primaries(method, *command, **kwargs, &block) if replica_disabled?
 
         replica_node_keys = @replications.values.map(&:sample)
-        try_map do |node_key, client|
+        results, errors = try_map do |node_key, client|
           next if primary?(node_key) || !replica_node_keys.include?(node_key)
 
           client.send(method, *command, **kwargs, &block)
-        end.values
+        end
+
+        return results.values if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
+      end
+
+      def send_ping(method, *command, **kwargs, &block)
+        results, errors = try_map do |_, client|
+          client.send(method, *command, **kwargs, &block)
+        end
+
+        return results.values if errors.empty?
+
+        raise ReloadNeeded if errors.values.any?(::RedisClient::ConnectionError)
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
       def scale_reading_clients
@@ -144,14 +185,9 @@ class RedisClient
         end
       end
 
-      def slot_exists?(slot)
-        slot = Integer(slot)
-        return false if slot < MIN_SLOT || slot > MAX_SLOT
-
-        !@slots[slot].nil?
-      end
-
       def find_node_key_of_primary(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -159,6 +195,8 @@ class RedisClient
       end
 
       def find_node_key_of_replica(slot)
+        return if slot.nil?
+
         slot = Integer(slot)
         return if slot < MIN_SLOT || slot > MAX_SLOT
 
@@ -171,6 +209,12 @@ class RedisClient
         @mutex.synchronize { @slots[slot] = node_key }
       end
 
+      def replicated?(primary_node_key, replica_node_key)
+        return false if @replications.nil? || @replications.size.zero?
+
+        @replications.fetch(primary_node_key).include?(replica_node_key)
+      end
+
       private
 
       def replica_disabled?
@@ -182,7 +226,9 @@ class RedisClient
       end
 
       def replica?(node_key)
-        !(@replications.nil? || @replications.size.zero?) && !@replications.key?(node_key)
+        return false if @replications.nil? || @replications.size.zero?
+
+        !@replications.key?(node_key)
       end
 
       def build_slot_node_mappings(node_info)
@@ -196,11 +242,12 @@ class RedisClient
         slots
       end
 
-      def build_replication_mappings(node_info)
+      def build_replication_mappings(node_info) # rubocop:disable Metrics/AbcSize
         dict = node_info.to_h { |info| [info[:id], info] }
         node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
           primary_info = dict[info[:primary_id]]
           acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
+          acc[info[:node_key]] if info[:role] == 'master' # for the primary which have no replicas
         end
       end
 
@@ -219,22 +266,20 @@ class RedisClient
       end
 
       def try_map # rubocop:disable Metrics/MethodLength
-        errors = {}
         results = {}
+        errors = {}
         threads = @clients.map do |k, v|
           Thread.new(k, v) do |node_key, client|
             Thread.pass
             reply = yield(node_key, client)
             results[node_key] = reply unless reply.nil?
-          rescue ::RedisClient::CommandError => e
+          rescue StandardError => e
             errors[node_key] = e
           end
         end
 
         threads.each(&:join)
-        return results if errors.empty?
-
-        raise ::RedisClient::Cluster::CommandErrorCollection, errors
+        [results, errors]
       end
     end
   end

data/lib/redis_client/cluster.rb

@@ -40,8 +40,10 @@ class RedisClient
         @size.zero?
       end
 
-      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37 handle redirections
+      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         all_replies = Array.new(@size)
+        errors = {}
         threads = @grouped.map do |k, v|
           Thread.new(@client, k, v) do |client, node_key, rows|
             Thread.pass
@@ -59,11 +61,15 @@ class RedisClient
             raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
 
             rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
+          rescue StandardError => e
+            errors[node_key] = e
           end
         end
 
         threads.each(&:join)
-        all_replies
+        return all_replies if errors.empty?
+
+        raise ::RedisClient::Cluster::ErrorCollection, errors
       end
     end
 
@@ -164,8 +170,10 @@ class RedisClient
     end
 
     def close
-      @node.each(&:close)
+      @node.call_all(:close)
       nil
+    rescue StandardError
+      # ignore
     end
 
     private
@@ -184,10 +192,11 @@ class RedisClient
       when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
         @node.call_all(method, *command, **kwargs, &block).first
       when 'flushall', 'flushdb'
-        @node.call_primary(method, *command, **kwargs, &block).first
-      when 'wait'     then @node.call_primary(method, *command, **kwargs, &block).sum
-      when 'keys'     then @node.call_replica(method, *command, **kwargs, &block).flatten.sort
-      when 'dbsize'   then @node.call_replica(method, *command, **kwargs, &block).sum
+        @node.call_primaries(method, *command, **kwargs, &block).first
+      when 'ping'     then @node.send_ping(method, *command, **kwargs, &block).first
+      when 'wait'     then send_wait_command(method, *command, **kwargs, &block)
+      when 'keys'     then @node.call_replicas(method, *command, **kwargs, &block).flatten.sort
+      when 'dbsize'   then @node.call_replicas(method, *command, **kwargs, &block).sum
       when 'scan'     then _scan(*command, **kwargs)
       when 'lastsave' then @node.call_all(method, *command, **kwargs, &block).sort
       when 'role'     then @node.call_all(method, *command, **kwargs, &block)
@@ -205,6 +214,22 @@ class RedisClient
         node = assign_node(*command)
         try_send(node, method, *command, **kwargs, &block)
       end
+    rescue RedisClient::Cluster::Node::ReloadNeeded
+      update_cluster_info!
+      raise ::RedisClient::Cluster::NodeMightBeDown
+    end
+
+    def send_wait_command(method, *command, retry_count: 3, **kwargs, &block)
+      @node.call_primaries(method, *command, **kwargs, &block).sum
+    rescue RedisClient::Cluster::ErrorCollection => e
+      raise if retry_count <= 0
+      raise if e.errors.values.none? do |err|
+        err.message.include?('WAIT cannot be used with replica instances')
+      end
+
+      update_cluster_info!
+      retry_count -= 1
+      retry
     end
 
     def send_config_command(method, *command, **kwargs, &block)
@@ -232,13 +257,17 @@ class RedisClient
       end
     end
 
-    def send_cluster_command(method, *command, **kwargs, &block)
+    def send_cluster_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/MethodLength
       subcommand = command[1].to_s.downcase
       case subcommand
       when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
            'reset', 'set-config-epoch', 'setslot'
         raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, ['cluster', subcommand]
       when 'saveconfig' then @node.call_all(method, *command, **kwargs, &block).first
+      when 'getkeysinslot'
+        raise ArgumentError, command.join(' ') if command.size != 4
+
+        find_node(@node.find_node_key_of_replica(command[2])).send(method, *command, **kwargs, &block)
       else assign_node(*command).send(method, *command, **kwargs, &block)
       end
     end
@@ -248,7 +277,7 @@ class RedisClient
       when 'debug', 'kill'
         @node.call_all(method, *command, **kwargs, &block).first
       when 'flush', 'load'
-        @node.call_primary(method, *command, **kwargs, &block).first
+        @node.call_primaries(method, *command, **kwargs, &block).first
       else assign_node(*command).send(method, *command, **kwargs, &block)
       end
     end
@@ -266,18 +295,16 @@ class RedisClient
 
     # @see https://redis.io/topics/cluster-spec#redirection-and-resharding
     #   Redirection and resharding
-    def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       node.send(method, *args, **kwargs, &block)
     rescue ::RedisClient::CommandError => e
-      if e.message.start_with?(REPLY_MOVED)
-        raise if retry_count <= 0
+      raise if retry_count <= 0
 
+      if e.message.start_with?(REPLY_MOVED)
         node = assign_redirection_node(e.message)
         retry_count -= 1
         retry
       elsif e.message.start_with?(REPLY_ASK)
-        raise if retry_count <= 0
-
         node = assign_asking_node(e.message)
         node.call(CMD_ASKING)
         retry_count -= 1
@@ -286,8 +313,11 @@ class RedisClient
         raise
       end
     rescue ::RedisClient::ConnectionError
+      raise if retry_count <= 0
+
       update_cluster_info!
-      raise
+      retry_count -= 1
+      retry
     end
 
     def _scan(*command, **kwargs) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
@@ -329,41 +359,30 @@ class RedisClient
       find_node(node_key)
     end
 
-    def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
+    def find_node_key(*command, primary_only: false)
       key = @command.extract_first_key(command)
-      if key.empty?
-        return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
-
-        return @node.replica_node_keys.sample
-      end
-
-      slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
-      return unless @node.slot_exists?(slot)
+      slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
 
       if @command.should_send_to_primary?(command) || primary_only
-        @node.find_node_key_of_primary(slot)
+        @node.find_node_key_of_primary(slot) || @node.primary_node_keys.sample
       else
-        @node.find_node_key_of_replica(slot)
+        @node.find_node_key_of_replica(slot) || @node.replica_node_keys.sample
       end
     end
 
-    def find_node(node_key)
-      return @node.sample if node_key.nil?
-
+    def find_node(node_key, retry_count: 3)
       @node.find_by(node_key)
     rescue ::RedisClient::Cluster::Node::ReloadNeeded
-      update_cluster_info!(node_key)
-      @node.find_by(node_key)
+      raise ::RedieClient::Cluster::NodeMightBeDown if retry_count <= 0
+
+      update_cluster_info!
+      retry_count -= 1
+      retry
     end
 
-    def update_cluster_info!(node_key = nil)
+    def update_cluster_info!
       @mutex.synchronize do
-        unless node_key.nil?
-          host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
-          @config.add_node(host, port)
-        end
-
-        @node.each(&:close)
+        close
         @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.8
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-18 00:00:00.000000000 Z
+date: 2022-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client