redis-cluster-client 0.11.0 → 0.13.5

data/lib/redis_client/cluster/pipeline.rb

@@ -2,6 +2,7 @@
 
 require 'redis_client'
 require 'redis_client/cluster/errors'
+require 'redis_client/cluster/noop_command_builder'
 require 'redis_client/connection_mixin'
 require 'redis_client/middlewares'
 require 'redis_client/pooled'
@@ -55,32 +56,48 @@ class RedisClient
           results = Array.new(commands.size)
           @pending_reads += size
           write_multi(commands)
+          redirection_indices = stale_cluster_state = first_exception = nil
 
-          redirection_indices = nil
-          first_exception = nil
           size.times do |index|
             timeout = timeouts && timeouts[index]
-            result = read(timeout)
+            result = read(connection_timeout(timeout))
             @pending_reads -= 1
+
             if result.is_a?(::RedisClient::Error)
               result._set_command(commands[index])
+              result._set_config(config)
+
               if result.is_a?(::RedisClient::CommandError) && result.message.start_with?('MOVED', 'ASK')
                 redirection_indices ||= []
                 redirection_indices << index
               elsif exception
                 first_exception ||= result
               end
+
+              stale_cluster_state = true if result.message.start_with?('CLUSTERDOWN')
             end
+
             results[index] = result
           end
 
-          raise first_exception if exception && first_exception
-          return results if redirection_indices.nil?
+          if redirection_indices
+            err = ::RedisClient::Cluster::Pipeline::RedirectionNeeded.new
+            err.replies = results
+            err.indices = redirection_indices
+            err.first_exception = first_exception
+            raise err
+          end
+
+          if stale_cluster_state
+            err = ::RedisClient::Cluster::Pipeline::StaleClusterState.new
+            err.replies = results
+            err.first_exception = first_exception
+            raise err
+          end
+
+          raise first_exception if first_exception
 
-          err = ::RedisClient::Cluster::Pipeline::RedirectionNeeded.new
-          err.replies = results
-          err.indices = redirection_indices
-          raise err
+          results
         end
       end
 
@@ -92,10 +109,14 @@ class RedisClient
         end
       end
 
-      ReplySizeError = Class.new(::RedisClient::Error)
+      ReplySizeError = Class.new(::RedisClient::Cluster::Error)
 
-      class RedirectionNeeded < ::RedisClient::Error
-        attr_accessor :replies, :indices
+      class StaleClusterState < ::RedisClient::Cluster::Error
+        attr_accessor :replies, :first_exception
+      end
+
+      class RedirectionNeeded < ::RedisClient::Cluster::Error
+        attr_accessor :replies, :indices, :first_exception
       end
 
       def initialize(router, command_builder, concurrent_worker, exception:, seed: Random.new_seed)
@@ -162,14 +183,18 @@ class RedisClient
           end
         end
 
-        all_replies = errors = required_redirections = nil
+        all_replies = errors = required_redirections = cluster_state_errors = nil
 
         work_group.each do |node_key, v|
           case v
           when ::RedisClient::Cluster::Pipeline::RedirectionNeeded
             required_redirections ||= {}
             required_redirections[node_key] = v
+          when ::RedisClient::Cluster::Pipeline::StaleClusterState
+            cluster_state_errors ||= {}
+            cluster_state_errors[node_key] = v
           when StandardError
+            cluster_state_errors ||= {} if v.is_a?(::RedisClient::ConnectionError)
             errors ||= {}
             errors[node_key] = v
           else
@@ -179,15 +204,25 @@ class RedisClient
         end
 
         work_group.close
-        raise ::RedisClient::Cluster::ErrorCollection, errors unless errors.nil?
+        @router.renew_cluster_state if cluster_state_errors
+        raise ::RedisClient::Cluster::ErrorCollection.with_errors(errors).with_config(@router.config) unless errors.nil?
 
         required_redirections&.each do |node_key, v|
+          raise v.first_exception if v.first_exception
+
           all_replies ||= Array.new(@size)
           pipeline = @pipelines[node_key]
           v.indices.each { |i| v.replies[i] = handle_redirection(v.replies[i], pipeline, i) }
           pipeline.outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] }
         end
 
+        cluster_state_errors&.each do |node_key, v|
+          raise v.first_exception if v.first_exception
+
+          all_replies ||= Array.new(@size)
+          @pipelines[node_key].outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] }
+        end
+
         all_replies
       end
 
@@ -195,7 +230,7 @@ class RedisClient
 
       def append_pipeline(node_key)
         @pipelines ||= {}
-        @pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(@command_builder)
+        @pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(::RedisClient::Cluster::NoopCommandBuilder)
         @pipelines[node_key].add_outer_index(@size)
         @size += 1
         @pipelines[node_key]
@@ -248,14 +283,14 @@ class RedisClient
         args = timeout.nil? ? [] : [timeout]
 
         if block.nil?
-          @router.try_send(node, method, command, args)
+          @router.send_command_to_node(node, method, command, args)
         else
-          @router.try_send(node, method, command, args, &block)
+          @router.send_command_to_node(node, method, command, args, &block)
         end
       end
 
       def try_asking(node)
-        node.call('ASKING') == 'OK'
+        node.call('asking') == 'OK'
       rescue StandardError
         false
       end

data/lib/redis_client/cluster/pub_sub.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'redis_client'
-require 'redis_client/cluster/normalized_cmd_name'
+require 'redis_client/cluster/errors'
 
 class RedisClient
   class Cluster
@@ -24,6 +24,8 @@ class RedisClient
         def close
           @worker.exit if @worker&.alive?
           @client.close
+        rescue ::RedisClient::ConnectionError
+          # ignore
         end
 
         private
@@ -32,11 +34,15 @@ class RedisClient
           # Ruby VM allocates 1 MB memory as a stack for a thread.
           # It is a fixed size but we can modify the size with some environment variables.
           # So it consumes memory 1 MB multiplied a number of workers.
-          Thread.new(client, queue) do |pubsub, q|
+          Thread.new(client, queue, nil) do |pubsub, q, prev_err|
             loop do
               q << pubsub.next_event
+              prev_err = nil
             rescue StandardError => e
+              next sleep 0.005 if e.instance_of?(prev_err.class) && e.message == prev_err&.message
+
               q << e
+              prev_err = e
             end
           end
         end
@@ -44,32 +50,40 @@ class RedisClient
 
       BUF_SIZE = Integer(ENV.fetch('REDIS_CLIENT_PUBSUB_BUF_SIZE', 1024))
 
+      private_constant :BUF_SIZE
+
       def initialize(router, command_builder)
         @router = router
         @command_builder = command_builder
         @queue = SizedQueue.new(BUF_SIZE)
         @state_dict = {}
+        @commands = []
       end
 
       def call(*args, **kwargs)
-        _call(@command_builder.generate(args, kwargs))
+        command = @command_builder.generate(args, kwargs)
+        _call(command)
+        @commands << command
         nil
       end
 
       def call_v(command)
-        _call(@command_builder.generate(command))
+        command = @command_builder.generate(command)
+        _call(command)
+        @commands << command
         nil
       end
 
       def close
         @state_dict.each_value(&:close)
         @state_dict.clear
+        @commands.clear
         @queue.clear
         @queue.close
         nil
       end
 
-      def next_event(timeout = nil)
+      def next_event(timeout = nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
         @state_dict.each_value(&:ensure_worker)
         max_duration = calc_max_duration(timeout)
         starting = obtain_current_time
@@ -78,6 +92,11 @@ class RedisClient
           break if max_duration > 0 && obtain_current_time - starting > max_duration
 
           case event = @queue.pop(true)
+          when ::RedisClient::CommandError
+            raise event unless event.message.start_with?('MOVED', 'CLUSTERDOWN')
+
+            break start_over
+          when ::RedisClient::ConnectionError then break start_over
           when StandardError then raise event
           when Array then break event
           end
@@ -88,22 +107,39 @@ class RedisClient
 
       private
 
-      def _call(command)
-        case ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_command(command)
-        when 'subscribe', 'psubscribe', 'ssubscribe' then call_to_single_state(command)
-        when 'unsubscribe', 'punsubscribe' then call_to_all_states(command)
-        when 'sunsubscribe' then call_for_sharded_states(command)
-        else call_to_single_state(command)
+      def _call(command) # rubocop:disable Metrics/AbcSize
+        if command.first.casecmp('subscribe').zero?
+          call_to_single_state(command)
+        elsif command.first.casecmp('psubscribe').zero?
+          call_to_single_state(command)
+        elsif command.first.casecmp('ssubscribe').zero?
+          call_to_single_state(command)
+        elsif command.first.casecmp('unsubscribe').zero?
+          call_to_all_states(command)
+        elsif command.first.casecmp('punsubscribe').zero?
+          call_to_all_states(command)
+        elsif command.first.casecmp('sunsubscribe').zero?
+          call_for_sharded_states(command)
+        else
+          call_to_single_state(command)
         end
       end
 
       def call_to_single_state(command)
         node_key = @router.find_node_key(command)
-        try_call(node_key, command)
+
+        handle_connection_error(node_key) do
+          @state_dict[node_key] ||= State.new(@router.find_node(node_key).pubsub, @queue)
+          @state_dict[node_key].call(command)
+        end
       end
 
       def call_to_all_states(command)
-        @state_dict.each_value { |s| s.call(command) }
+        @state_dict.each do |node_key, state|
+          handle_connection_error(node_key, ignore: true) do
+            state.call(command)
+          end
+        end
       end
 
       def call_for_sharded_states(command)
@@ -114,24 +150,6 @@ class RedisClient
         end
       end
 
-      def try_call(node_key, command, retry_count: 1)
-        add_state(node_key).call(command)
-      rescue ::RedisClient::CommandError => e
-        raise if !e.message.start_with?('MOVED') || retry_count <= 0
-
-        # for sharded pub/sub
-        node_key = e.message.split[2]
-        retry_count -= 1
-        retry
-      end
-
-      def add_state(node_key)
-        return @state_dict[node_key] if @state_dict.key?(node_key)
-
-        state = State.new(@router.find_node(node_key).pubsub, @queue)
-        @state_dict[node_key] = state
-      end
-
       def obtain_current_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
       end
@@ -139,6 +157,27 @@ class RedisClient
       def calc_max_duration(timeout)
         timeout.nil? || timeout < 0 ? 0 : timeout * 1_000_000
       end
+
+      def handle_connection_error(node_key, ignore: false)
+        yield
+      rescue ::RedisClient::ConnectionError
+        @state_dict[node_key]&.close
+        @state_dict.delete(node_key)
+        @router.renew_cluster_state
+        raise unless ignore
+      end
+
+      def start_over
+        loop do
+          @router.renew_cluster_state
+          @state_dict.each_value(&:close)
+          @state_dict.clear
+          @commands.each { |command| _call(command) }
+          break
+        rescue ::RedisClient::ConnectionError, ::RedisClient::Cluster::NodeMightBeDown
+          sleep 1.0
+        end
+      end
     end
   end
 end