redis-cluster-client 0.3.5 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a697c2bd33f615cb435ee48c11c301bba56cad4eac139f30db70a183d888d891
-  data.tar.gz: 52200e9400bd7bd7fde2019c52b6cf1e26c6c019cbf1ed12a25c2167cd7ac129
+  metadata.gz: 5243f77ab4cb1322ddaaade9b001780e8f24d702d293f56870ce9e1109780222
+  data.tar.gz: fabae20f3a0d9bede270602adb247ca3cbe2b36ad42d68ecdd908320b18a31bb
 SHA512:
-  metadata.gz: 05dc86fd1aa3cbf3cd3f0fa43d9d1c8463fcbe9e0ce1bcfe094f6095fd92d878a4bb21e6d6fa62203e91f3dc642e5869552f0b83d579cfa158b791501e1cb600
-  data.tar.gz: 8f80c5072d4d1bdbeb3e2cb4c9f92493537a6d9002edd71426598844ba28a497171c055135bb19a1d89a325c194026195100f9a0aea89c0beac3d8e9424f7f58
+  metadata.gz: 999a6b334092b0e498d5e8ba773131542d212d8e5b8cb7b117811c465ccb4385b5bea11637cd8e4febdebc3d1e33c218b084e13bd5e5f1686ab477dca0f9411f
+  data.tar.gz: b7edede3965d1878459a128a3e7cc0f5c84b9b3e434ec44c86b408830670058b28ec023ffeeb2f0bcd1503ba5c274c46ecf791f5b03ebbdcccff09c3e3b1cf06

data/lib/redis_client/cluster/command.rb

@@ -10,7 +10,7 @@ class RedisClient
       EMPTY_STRING = ''
 
       class << self
-        def load(nodes) # rubocop:disable Metrics/MethodLength
+        def load(nodes)
           errors = []
           cmd = nil
           nodes&.each do |node|
@@ -32,10 +32,7 @@ class RedisClient
 
         def parse_command_details(rows)
           rows&.reject { |row| row[0].nil? }.to_h do |row|
-            [
-              ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_name(row[0]),
-              { arity: row[1], flags: row[2], first: row[3], last: row[4], step: row[5] }
-            ]
+            [row[0].downcase, { arity: row[1], flags: row[2], first: row[3], last: row[4], step: row[5] }]
           end
         end
       end
@@ -85,7 +82,7 @@ class RedisClient
         @details.fetch(name).fetch(key)
       end
 
-      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity
         case ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_command(command)
         when 'eval', 'evalsha', 'zinterstore', 'zunionstore' then 3
         when 'object' then 2

data/lib/redis_client/cluster/node/latency_replica.rb

@@ -34,14 +34,13 @@ class RedisClient
 
         def any_replica_node_key(seed: nil)
           random = seed.nil? ? Random : Random.new(seed)
-          @existed_replicas.sample(random: random).first
+          @existed_replicas.sample(random: random)&.first
         end
 
         private
 
-        def measure_latencies(clients) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-          latencies = nil
-          clients.each_slice(::RedisClient::Cluster::Node::MAX_THREADS) do |chuncked_clients|
+        def measure_latencies(clients) # rubocop:disable Metrics/AbcSize
+          clients.each_slice(::RedisClient::Cluster::Node::MAX_THREADS).each_with_object({}) do |chuncked_clients, acc|
             threads = chuncked_clients.map do |k, v|
               Thread.new(k, v) do |node_key, client|
                 Thread.pass
@@ -63,12 +62,9 @@ class RedisClient
 
             threads.each do |t|
               t.join
-              latencies ||= {}
-              latencies[t.thread_variable_get(:node_key)] = t.thread_variable_get(:latency)
+              acc[t.thread_variable_get(:node_key)] = t.thread_variable_get(:latency)
             end
           end
-
-          latencies
         end
 
         def select_replica_clients(replications, clients)

data/lib/redis_client/cluster/node.rb

@@ -37,7 +37,7 @@ class RedisClient
       end
 
       class << self
-        def load_info(options, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def load_info(options, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
           startup_size = options.size > MAX_STARTUP_SAMPLE ? MAX_STARTUP_SAMPLE : options.size
           node_info_list = errors = nil
           startup_options = options.to_a.sample(MAX_STARTUP_SAMPLE).to_h
@@ -83,7 +83,7 @@ class RedisClient
 
         # @see https://redis.io/commands/cluster-nodes/
         # @see https://github.com/redis/redis/blob/78960ad57b8a5e6af743d789ed8fd767e37d42b8/src/cluster.c#L4660-L4683
-        def parse_node_info(info) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+        def parse_node_info(info) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
           rows = info.split("\n").map(&:split)
           rows.each { |arr| arr[2] = arr[2].split(',') }
           rows.select! { |arr| arr[7] == 'connected' && (arr[2] & %w[fail? fail handshake noaddr noflags]).empty? }
@@ -232,7 +232,7 @@ class RedisClient
           client.send(method, *args, command, &block)
         end
 
-        [results.values, errors]
+        [results&.values, errors]
       end
 
       def call_multiple_nodes!(clients, method, command, args, &block)
@@ -242,7 +242,7 @@ class RedisClient
         raise ::RedisClient::Cluster::ErrorCollection, errors
       end
 
-      def try_map(clients) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def try_map(clients) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         results = errors = nil
         clients.each_slice(MAX_THREADS) do |chuncked_clients|
           threads = chuncked_clients.map do |k, v|

data/lib/redis_client/cluster/normalized_cmd_name.rb

@@ -28,6 +28,7 @@ class RedisClient
 
       def clear
         @mutex.synchronize { @cache.clear }
+        true
       end
 
       private

data/lib/redis_client/cluster/pipeline.rb

@@ -2,84 +2,172 @@
 
 require 'redis_client'
 require 'redis_client/cluster/errors'
+require 'redis_client/connection_mixin'
+require 'redis_client/middlewares'
+require 'redis_client/pooled'
 
 class RedisClient
   class Cluster
     class Pipeline
+      class Extended < ::RedisClient::Pipeline
+        attr_reader :outer_indices
+
+        def initialize(command_builder)
+          super
+          @outer_indices = nil
+        end
+
+        def add_outer_index(index)
+          @outer_indices ||= []
+          @outer_indices << index
+        end
+
+        def get_inner_index(outer_index)
+          @outer_indices&.find_index(outer_index)
+        end
+
+        def get_callee_method(inner_index)
+          if @timeouts.is_a?(Array) && !@timeouts[inner_index].nil?
+            :blocking_call_v
+          elsif _retryable?
+            :call_once_v
+          else
+            :call_v
+          end
+        end
+
+        def get_command(inner_index)
+          @commands.is_a?(Array) ? @commands[inner_index] : nil
+        end
+
+        def get_timeout(inner_index)
+          @timeouts.is_a?(Array) ? @timeouts[inner_index] : nil
+        end
+
+        def get_block(inner_index)
+          @blocks.is_a?(Array) ? @blocks[inner_index] : nil
+        end
+      end
+
+      ::RedisClient::ConnectionMixin.module_eval do
+        def call_pipelined_aware_of_redirection(commands, timeouts) # rubocop:disable Metrics/AbcSize
+          size = commands.size
+          results = Array.new(commands.size)
+          @pending_reads += size
+          write_multi(commands)
+
+          redirection_indices = nil
+          size.times do |index|
+            timeout = timeouts && timeouts[index]
+            result = read(timeout)
+            @pending_reads -= 1
+            if result.is_a?(CommandError)
+              result._set_command(commands[index])
+              if result.message.start_with?('MOVED', 'ASK')
+                redirection_indices ||= []
+                redirection_indices << index
+              end
+            end
+
+            results[index] = result
+          end
+
+          return results if redirection_indices.nil?
+
+          err = ::RedisClient::Cluster::Pipeline::RedirectionNeeded.new
+          err.replies = results
+          err.indices = redirection_indices
+          raise err
+        end
+      end
+
       ReplySizeError = Class.new(::RedisClient::Error)
+
+      class RedirectionNeeded < ::RedisClient::Error
+        attr_accessor :replies, :indices
+      end
+
       MAX_THREADS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', 5))
 
       def initialize(router, command_builder, seed: Random.new_seed)
         @router = router
         @command_builder = command_builder
-        @grouped = {}
-        @size = 0
         @seed = seed
+        @pipelines = nil
+        @size = 0
       end
 
       def call(*args, **kwargs, &block)
         command = @command_builder.generate(args, kwargs)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :call_v, command, block])
+        append_pipeline(node_key).call_v(command, &block)
       end
 
       def call_v(args, &block)
         command = @command_builder.generate(args)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :call_v, command, block])
+        append_pipeline(node_key).call_v(command, &block)
       end
 
       def call_once(*args, **kwargs, &block)
         command = @command_builder.generate(args, kwargs)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :call_once_v, command, block])
+        append_pipeline(node_key).call_once_v(command, &block)
       end
 
       def call_once_v(args, &block)
         command = @command_builder.generate(args)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :call_once_v, command, block])
+        append_pipeline(node_key).call_once_v(command, &block)
       end
 
       def blocking_call(timeout, *args, **kwargs, &block)
         command = @command_builder.generate(args, kwargs)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :blocking_call_v, timeout, command, block])
+        append_pipeline(node_key).blocking_call_v(timeout, command, &block)
       end
 
       def blocking_call_v(timeout, args, &block)
         command = @command_builder.generate(args)
         node_key = @router.find_node_key(command, seed: @seed)
-        add_row(node_key, [@size, :blocking_call_v, timeout, command, block])
+        append_pipeline(node_key).blocking_call_v(timeout, command, &block)
       end
 
       def empty?
         @size.zero?
       end
 
-      # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37 handle redirections
-      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         all_replies = errors = nil
-        @grouped.each_slice(MAX_THREADS) do |chuncked_grouped|
-          threads = chuncked_grouped.map do |k, v|
-            Thread.new(@router, k, v) do |router, node_key, rows|
+        @pipelines&.each_slice(MAX_THREADS) do |chuncked_pipelines|
+          threads = chuncked_pipelines.map do |node_key, pipeline|
+            Thread.new(node_key, pipeline) do |nk, pl|
               Thread.pass
-              replies = do_pipelining(router.find_node(node_key), rows)
-              raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
+              Thread.current.thread_variable_set(:node_key, nk)
+              replies = do_pipelining(@router.find_node(nk), pl)
+              raise ReplySizeError, "commands: #{pl._size}, replies: #{replies.size}" if pl._size != replies.size
 
-              Thread.current.thread_variable_set(:rows, rows)
               Thread.current.thread_variable_set(:replies, replies)
+            rescue ::RedisClient::Cluster::Pipeline::RedirectionNeeded => e
+              Thread.current.thread_variable_set(:redirection_needed, e)
             rescue StandardError => e
-              Thread.current.thread_variable_set(:node_key, node_key)
               Thread.current.thread_variable_set(:error, e)
             end
           end
 
+          threads.each(&:join)
           threads.each do |t|
-            t.join
             if t.thread_variable?(:replies)
               all_replies ||= Array.new(@size)
-              t.thread_variable_get(:rows).each_with_index { |r, i| all_replies[r.first] = t.thread_variable_get(:replies)[i] }
+              @pipelines[t.thread_variable_get(:node_key)]
+                .outer_indices
+                .each_with_index { |outer, inner| all_replies[outer] = t.thread_variable_get(:replies)[inner] }
+            elsif t.thread_variable?(:redirection_needed)
+              all_replies ||= Array.new(@size)
+              pipeline = @pipelines[t.thread_variable_get(:node_key)]
+              err = t.thread_variable_get(:redirection_needed)
+              err.indices.each { |i| err.replies[i] = handle_redirection(err.replies[i], pipeline, i) }
+              pipeline.outer_indices.each_with_index { |outer, inner| all_replies[outer] = err.replies[inner] }
             elsif t.thread_variable?(:error)
               errors ||= {}
               errors[t.thread_variable_get(:node_key)] = t.thread_variable_get(:error)
@@ -87,28 +175,78 @@ class RedisClient
           end
         end
 
-        return all_replies if errors.nil?
+        raise ::RedisClient::Cluster::ErrorCollection, errors unless errors.nil?
 
-        raise ::RedisClient::Cluster::ErrorCollection, errors
+        all_replies
       end
 
       private
 
-      def add_row(node_key, row)
-        @grouped[node_key] = [] unless @grouped.key?(node_key)
-        @grouped[node_key] << row
+      def append_pipeline(node_key)
+        @pipelines ||= {}
+        @pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(@command_builder)
+        @pipelines[node_key].add_outer_index(@size)
         @size += 1
+        @pipelines[node_key]
       end
 
-      def do_pipelining(node, rows)
-        node.pipelined do |pipeline|
-          rows.each do |row|
-            case row.size
-            when 4 then pipeline.send(row[1], row[2], &row[3])
-            when 5 then pipeline.send(row[1], row[2], row[3], &row[4])
-            end
+      def do_pipelining(client, pipeline)
+        case client
+        when ::RedisClient then send_pipeline(client, pipeline)
+        when ::RedisClient::Pooled then client.with { |cli| send_pipeline(cli, pipeline) }
+        else raise NotImplementedError, "#{client.class.name}#pipelined for cluster client"
+        end
+      end
+
+      def send_pipeline(client, pipeline)
+        results = client.send(:ensure_connected, retryable: pipeline._retryable?) do |connection|
+          commands = pipeline._commands
+          ::RedisClient::Middlewares.call_pipelined(commands, client.config) do
+            connection.call_pipelined_aware_of_redirection(commands, pipeline._timeouts)
           end
         end
+
+        pipeline._coerce!(results)
+      end
+
+      def handle_redirection(err, pipeline, inner_index)
+        return err unless err.is_a?(::RedisClient::CommandError)
+
+        if err.message.start_with?('MOVED')
+          node = @router.assign_redirection_node(err.message)
+          try_redirection(node, pipeline, inner_index)
+        elsif err.message.start_with?('ASK')
+          node = @router.assign_asking_node(err.message)
+          try_asking(node) ? try_redirection(node, pipeline, inner_index) : err
+        else
+          err
+        end
+      end
+
+      def try_redirection(node, pipeline, inner_index)
+        redirect_command(node, pipeline, inner_index)
+      rescue StandardError => e
+        e
+      end
+
+      def redirect_command(node, pipeline, inner_index)
+        method = pipeline.get_callee_method(inner_index)
+        command = pipeline.get_command(inner_index)
+        timeout = pipeline.get_timeout(inner_index)
+        block = pipeline.get_block(inner_index)
+        args = timeout.nil? ? [] : [timeout]
+
+        if block.nil?
+          @router.try_send(node, method, command, args)
+        else
+          @router.try_send(node, method, command, args, &block)
+        end
+      end
+
+      def try_asking(node)
+        node.call('ASKING') == 'OK'
+      rescue StandardError
+        false
       end
     end
   end

data/lib/redis_client/cluster/router.rb

@@ -25,7 +25,7 @@ class RedisClient
         @command_builder = @config.command_builder
       end
 
-      def send_command(method, command, *args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def send_command(method, command, *args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         cmd = ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_command(command)
         case cmd
         when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
@@ -65,7 +65,7 @@ class RedisClient
 
       # @see https://redis.io/topics/cluster-spec#redirection-and-resharding
       #   Redirection and resharding
-      def try_send(node, method, command, args, retry_count: 3, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def try_send(node, method, command, args, retry_count: 3, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         if args.empty?
           # prevent memory allocation for variable-length args
           node.send(method, command, &block)
@@ -100,7 +100,7 @@ class RedisClient
         retry
       end
 
-      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize
         node.send(method, *args, **kwargs, &block)
       rescue ::RedisClient::CommandError => e
         raise if retry_count <= 0
@@ -129,7 +129,7 @@ class RedisClient
         retry
       end
 
-      def scan(*command, seed: nil, **kwargs) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def scan(*command, seed: nil, **kwargs) # rubocop:disable Metrics/AbcSize
         command = @command_builder.generate(command, kwargs)
 
         command[1] = ZERO_CURSOR_FOR_SCAN if command.size == 1
@@ -172,7 +172,7 @@ class RedisClient
       def find_node(node_key, retry_count: 3)
         @node.find_by(node_key)
       rescue ::RedisClient::Cluster::Node::ReloadNeeded
-        raise ::RedieClient::Cluster::NodeMightBeDown if retry_count <= 0
+        raise ::RedisClient::Cluster::NodeMightBeDown if retry_count <= 0
 
         update_cluster_info!
         retry_count -= 1
@@ -183,6 +183,18 @@ class RedisClient
         @command.exists?(name)
       end
 
+      def assign_redirection_node(err_msg)
+        _, slot, node_key = err_msg.split
+        slot = slot.to_i
+        @node.update_slot(slot, node_key)
+        find_node(node_key)
+      end
+
+      def assign_asking_node(err_msg)
+        _, _, node_key = err_msg.split
+        find_node(node_key)
+      end
+
       private
 
       def send_wait_command(method, command, args, retry_count: 3, &block)
@@ -258,19 +270,7 @@ class RedisClient
         end
       end
 
-      def assign_redirection_node(err_msg)
-        _, slot, node_key = err_msg.split
-        slot = slot.to_i
-        @node.update_slot(slot, node_key)
-        find_node(node_key)
-      end
-
-      def assign_asking_node(err_msg)
-        _, _, node_key = err_msg.split
-        find_node(node_key)
-      end
-
-      def fetch_cluster_info(config, pool: nil, **kwargs) # rubocop:disable Metrics/MethodLength
+      def fetch_cluster_info(config, pool: nil, **kwargs)
         node_info = ::RedisClient::Cluster::Node.load_info(config.per_node_key, **kwargs)
         node_addrs = node_info.map { |info| ::RedisClient::Cluster::NodeKey.hashify(info[:node_key]) }
         config.update_node(node_addrs)

data/lib/redis_client/cluster.rb

@@ -81,7 +81,7 @@ class RedisClient
       seed = @config.use_replica? && @config.replica_affinity == :random ? nil : Random.new_seed
       pipeline = ::RedisClient::Cluster::Pipeline.new(@router, @command_builder, seed: seed)
       yield pipeline
-      return [] if pipeline.empty? == 0
+      return [] if pipeline.empty?
 
       pipeline.execute
     end

data/lib/redis_client/cluster_config.rb

@@ -114,7 +114,7 @@ class RedisClient
       end
     end
 
-    def parse_node_url(addr) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+    def parse_node_url(addr) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       return if addr.empty?
 
       uri = URI(addr)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.7
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-21 00:00:00.000000000 Z
+date: 2022-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client