redis-cluster-client 0.7.5 → 0.7.6

data/lib/redis_client/cluster/node.rb

@@ -2,6 +2,7 @@
 
 require 'redis_client'
 require 'redis_client/config'
+require 'redis_client/cluster/error_identification'
 require 'redis_client/cluster/errors'
 require 'redis_client/cluster/node/primary_only'
 require 'redis_client/cluster/node/random_replica'
@@ -22,10 +23,10 @@ class RedisClient
       SLOT_SIZE = 16_384
       MIN_SLOT = 0
       MAX_SLOT = SLOT_SIZE - 1
-      IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
       DEAD_FLAGS = %w[fail? fail handshake noaddr noflags].freeze
       ROLE_FLAGS = %w[master slave].freeze
       EMPTY_ARRAY = [].freeze
+      EMPTY_HASH = {}.freeze
 
       ReloadNeeded = Class.new(::RedisClient::Error)
 
@@ -78,9 +79,11 @@ class RedisClient
       end
 
       class Config < ::RedisClient::Config
-        def initialize(scale_read: false, **kwargs)
+        def initialize(scale_read: false, middlewares: nil, **kwargs)
           @scale_read = scale_read
-          super(**kwargs)
+          middlewares ||= []
+          middlewares.unshift ErrorIdentification::Middleware
+          super(middlewares: middlewares, **kwargs)
         end
 
         private
@@ -92,119 +95,19 @@ class RedisClient
         end
       end
 
-      class << self
-        def load_info(options, concurrent_worker, slow_command_timeout: -1, **kwargs) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          raise ::RedisClient::Cluster::InitialSetupError, [] if options.nil? || options.empty?
-
-          startup_size = options.size > MAX_STARTUP_SAMPLE ? MAX_STARTUP_SAMPLE : options.size
-          startup_options = options.to_a.sample(startup_size).to_h
-          startup_nodes = ::RedisClient::Cluster::Node.new(startup_options, concurrent_worker, **kwargs)
-          work_group = concurrent_worker.new_group(size: startup_size)
-
-          startup_nodes.each_with_index do |raw_client, i|
-            work_group.push(i, raw_client) do |client|
-              regular_timeout = client.read_timeout
-              client.read_timeout = slow_command_timeout > 0.0 ? slow_command_timeout : regular_timeout
-              reply = client.call('CLUSTER', 'NODES')
-              client.read_timeout = regular_timeout
-              parse_cluster_node_reply(reply)
-            rescue StandardError => e
-              e
-            ensure
-              client&.close
-            end
-          end
-
-          node_info_list = errors = nil
-
-          work_group.each do |i, v|
-            case v
-            when StandardError
-              errors ||= Array.new(startup_size)
-              errors[i] = v
-            else
-              node_info_list ||= Array.new(startup_size)
-              node_info_list[i] = v
-            end
-          end
-
-          work_group.close
-
-          raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.nil?
-
-          grouped = node_info_list.compact.group_by do |info_list|
-            info_list.sort_by!(&:id)
-            info_list.each_with_object(String.new(capacity: 128 * info_list.size)) do |e, a|
-              a << e.id << e.node_key << e.role << e.primary_id << e.config_epoch
-            end
-          end
-
-          grouped.max_by { |_, v| v.size }[1].first.freeze
-        end
-
-        private
-
-        # @see https://redis.io/commands/cluster-nodes/
-        # @see https://github.com/redis/redis/blob/78960ad57b8a5e6af743d789ed8fd767e37d42b8/src/cluster.c#L4660-L4683
-        def parse_cluster_node_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          reply.each_line("\n", chomp: true).filter_map do |line|
-            fields = line.split
-            flags = fields[2].split(',')
-            next unless fields[7] == 'connected' && (flags & DEAD_FLAGS).empty?
-
-            slots = if fields[8].nil?
-                      EMPTY_ARRAY
-                    else
-                      fields[8..].reject { |str| str.start_with?('[') }
-                                 .map { |str| str.split('-').map { |s| Integer(s) } }
-                                 .map { |a| a.size == 1 ? a << a.first : a }
-                                 .map(&:sort)
-                    end
-
-            ::RedisClient::Cluster::Node::Info.new(
-              id: fields[0],
-              node_key: parse_node_key(fields[1]),
-              role: (flags & ROLE_FLAGS).first,
-              primary_id: fields[3],
-              ping_sent: fields[4],
-              pong_recv: fields[5],
-              config_epoch: fields[6],
-              link_state: fields[7],
-              slots: slots
-            )
-          end
-        end
-
-        # As redirection node_key is dependent on `cluster-preferred-endpoint-type` config,
-        # node_key should use hostname if present in CLUSTER NODES output.
-        #
-        # See https://redis.io/commands/cluster-nodes/ for details on the output format.
-        # node_address matches fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]>
-        def parse_node_key(node_address)
-          ip_chunk, hostname, _auxiliaries = node_address.split(',')
-          ip_port_string = ip_chunk.split('@').first
-          return ip_port_string if hostname.nil? || hostname.empty?
-
-          port = ip_port_string.split(':')[1]
-          "#{hostname}:#{port}"
-        end
-      end
-
       def initialize(
-        options,
         concurrent_worker,
-        node_info_list: [],
-        with_replica: false,
-        replica_affinity: :random,
+        config:,
         pool: nil,
         **kwargs
       )
 
         @concurrent_worker = concurrent_worker
-        @slots = build_slot_node_mappings(node_info_list)
-        @replications = build_replication_mappings(node_info_list)
-        klass = make_topology_class(with_replica, replica_affinity)
-        @topology = klass.new(@replications, options, pool, @concurrent_worker, **kwargs)
+        @slots = build_slot_node_mappings(EMPTY_ARRAY)
+        @replications = build_replication_mappings(EMPTY_ARRAY)
+        klass = make_topology_class(config.use_replica?, config.replica_affinity)
+        @topology = klass.new(pool, @concurrent_worker, **kwargs)
+        @config = config
         @mutex = Mutex.new
       end
 
@@ -255,6 +158,14 @@ class RedisClient
         @topology.clients_for_scanning(seed: seed).values.sort_by { |c| "#{c.config.host}-#{c.config.port}" }
       end
 
+      def clients
+        @topology.clients.values
+      end
+
+      def primary_clients
+        @topology.primary_clients.values
+      end
+
       def replica_clients
         @topology.replica_clients.values
       end
@@ -292,6 +203,24 @@ class RedisClient
         end
       end
 
+      def reload!
+        with_reload_lock do
+          with_startup_clients(MAX_STARTUP_SAMPLE) do |startup_clients|
+            @node_info = refetch_node_info_list(startup_clients)
+            @node_configs = @node_info.to_h do |node_info|
+              [node_info.node_key, @config.client_config_for_node(node_info.node_key)]
+            end
+            @slots = build_slot_node_mappings(@node_info)
+            @replications = build_replication_mappings(@node_info)
+            @topology.process_topology_update!(@replications, @node_configs)
+          end
+        end
+      end
+
+      def owns_error?(err)
+        any? { |c| ErrorIdentification.client_owns_error?(err, c) }
+      end
+
       private
 
       def make_topology_class(with_replica, replica_affinity)
@@ -378,6 +307,137 @@ class RedisClient
 
         [results, errors]
       end
+
+      def refetch_node_info_list(startup_clients) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        startup_size = startup_clients.size
+        work_group = @concurrent_worker.new_group(size: startup_size)
+
+        startup_clients.each_with_index do |raw_client, i|
+          work_group.push(i, raw_client) do |client|
+            regular_timeout = client.read_timeout
+            client.read_timeout = @config.slow_command_timeout > 0.0 ? @config.slow_command_timeout : regular_timeout
+            reply = client.call('CLUSTER', 'NODES')
+            client.read_timeout = regular_timeout
+            parse_cluster_node_reply(reply)
+          rescue StandardError => e
+            e
+          ensure
+            client&.close
+          end
+        end
+
+        node_info_list = errors = nil
+
+        work_group.each do |i, v|
+          case v
+          when StandardError
+            errors ||= Array.new(startup_size)
+            errors[i] = v
+          else
+            node_info_list ||= Array.new(startup_size)
+            node_info_list[i] = v
+          end
+        end
+
+        work_group.close
+
+        raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.nil?
+
+        grouped = node_info_list.compact.group_by do |info_list|
+          info_list.sort_by!(&:id)
+          info_list.each_with_object(String.new(capacity: 128 * info_list.size)) do |e, a|
+            a << e.id << e.node_key << e.role << e.primary_id << e.config_epoch
+          end
+        end
+
+        grouped.max_by { |_, v| v.size }[1].first
+      end
+
+      def parse_cluster_node_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        reply.each_line("\n", chomp: true).filter_map do |line|
+          fields = line.split
+          flags = fields[2].split(',')
+          next unless fields[7] == 'connected' && (flags & DEAD_FLAGS).empty?
+
+          slots = if fields[8].nil?
+                    EMPTY_ARRAY
+                  else
+                    fields[8..].reject { |str| str.start_with?('[') }
+                               .map { |str| str.split('-').map { |s| Integer(s) } }
+                               .map { |a| a.size == 1 ? a << a.first : a }
+                               .map(&:sort)
+                  end
+
+          ::RedisClient::Cluster::Node::Info.new(
+            id: fields[0],
+            node_key: parse_node_key(fields[1]),
+            role: (flags & ROLE_FLAGS).first,
+            primary_id: fields[3],
+            ping_sent: fields[4],
+            pong_recv: fields[5],
+            config_epoch: fields[6],
+            link_state: fields[7],
+            slots: slots
+          )
+        end
+      end
+
+      # As redirection node_key is dependent on `cluster-preferred-endpoint-type` config,
+      # node_key should use hostname if present in CLUSTER NODES output.
+      #
+      # See https://redis.io/commands/cluster-nodes/ for details on the output format.
+      # node_address matches fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]>
+      def parse_node_key(node_address)
+        ip_chunk, hostname, _auxiliaries = node_address.split(',')
+        ip_port_string = ip_chunk.split('@').first
+        return ip_port_string if hostname.nil? || hostname.empty?
+
+        port = ip_port_string.split(':')[1]
+        "#{hostname}:#{port}"
+      end
+
+      def with_startup_clients(count) # rubocop:disable Metrics/AbcSize
+        if @config.connect_with_original_config
+          # If connect_with_original_config is set, that means we need to build actual client objects
+          # and close them, so that we e.g. re-resolve a DNS entry with the cluster nodes in it.
+          begin
+            # Memoize the startup clients, so we maintain RedisClient's internal circuit breaker configuration
+            # if it's set.
+            @startup_clients ||= @config.startup_nodes.values.sample(count).map do |node_config|
+              ::RedisClient::Cluster::Node::Config.new(**node_config).new_client
+            end
+            yield @startup_clients
+          ensure
+            # Close the startup clients when we're done, so we don't maintain pointless open connections to
+            # the cluster though
+            @startup_clients&.each(&:close)
+          end
+        else
+          # (re-)connect using nodes we already know about.
+          # If this is the first time we're connecting to the cluster, we need to seed the topology with the
+          # startup clients though.
+          @topology.process_topology_update!({}, @config.startup_nodes) if @topology.clients.empty?
+          yield @topology.clients.values.sample(count)
+        end
+      end
+
+      def with_reload_lock
+        # What should happen with concurrent calls #reload? This is a realistic possibility if the cluster goes into
+        # a CLUSTERDOWN state, and we're using a pooled backend. Every thread will independently discover this, and
+        # call reload!.
+        # For now, if a reload is in progress, wait for that to complete, and consider that the same as us having
+        # performed the reload.
+        # Probably in the future we should add a circuit breaker to #reload itself, and stop trying if the cluster is
+        # obviously not working.
+        wait_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @mutex.synchronize do
+          return if @last_reloaded_at && @last_reloaded_at > wait_start
+
+          r = yield
+          @last_reloaded_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          r
+        end
+      end
     end
   end
 end

data/lib/redis_client/cluster/pinning_node.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class RedisClient
+  class Cluster
+    class PinningNode
+      def initialize(client)
+        @client = client
+      end
+
+      def call(*args, **kwargs, &block)
+        @client.call(*args, **kwargs, &block)
+      end
+
+      def call_v(args, &block)
+        @client.call_v(args, &block)
+      end
+
+      def call_once(*args, **kwargs, &block)
+        @client.call_once(*args, **kwargs, &block)
+      end
+
+      def call_once_v(args, &block)
+        @client.call_once_v(args, &block)
+      end
+
+      def blocking_call(timeout, *args, **kwargs, &block)
+        @client.blocking_call(timeout, *args, **kwargs, &block)
+      end
+
+      def blocking_call_v(timeout, args, &block)
+        @client.blocking_call_v(timeout, args, &block)
+      end
+    end
+  end
+end

data/lib/redis_client/cluster/pipeline.rb

@@ -50,28 +50,31 @@ class RedisClient
       end
 
       ::RedisClient::ConnectionMixin.module_eval do
-        def call_pipelined_aware_of_redirection(commands, timeouts) # rubocop:disable Metrics/AbcSize
+        def call_pipelined_aware_of_redirection(commands, timeouts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
           size = commands.size
           results = Array.new(commands.size)
           @pending_reads += size
           write_multi(commands)
 
           redirection_indices = nil
+          exception = nil
           size.times do |index|
             timeout = timeouts && timeouts[index]
             result = read(timeout)
             @pending_reads -= 1
-            if result.is_a?(CommandError)
+            if result.is_a?(::RedisClient::Error)
               result._set_command(commands[index])
-              if result.message.start_with?('MOVED', 'ASK')
+              if result.is_a?(::RedisClient::CommandError) && result.message.start_with?('MOVED', 'ASK')
                 redirection_indices ||= []
                 redirection_indices << index
+              else
+                exception ||= result
               end
             end
-
             results[index] = result
           end
 
+          raise exception if exception
           return results if redirection_indices.nil?
 
           err = ::RedisClient::Cluster::Pipeline::RedirectionNeeded.new
@@ -158,15 +161,13 @@ class RedisClient
           end
         end
 
-        all_replies = errors = nil
+        all_replies = errors = required_redirections = nil
 
         work_group.each do |node_key, v|
           case v
           when ::RedisClient::Cluster::Pipeline::RedirectionNeeded
-            all_replies ||= Array.new(@size)
-            pipeline = @pipelines[node_key]
-            v.indices.each { |i| v.replies[i] = handle_redirection(v.replies[i], pipeline, i) }
-            pipeline.outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] }
+            required_redirections ||= {}
+            required_redirections[node_key] = v
           when StandardError
             errors ||= {}
             errors[node_key] = v
@@ -177,9 +178,15 @@ class RedisClient
         end
 
         work_group.close
-
         raise ::RedisClient::Cluster::ErrorCollection, errors unless errors.nil?
 
+        required_redirections&.each do |node_key, v|
+          all_replies ||= Array.new(@size)
+          pipeline = @pipelines[node_key]
+          v.indices.each { |i| v.replies[i] = handle_redirection(v.replies[i], pipeline, i) }
+          pipeline.outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] }
+        end
+
         all_replies
       end
 
@@ -217,21 +224,15 @@ class RedisClient
 
         if err.message.start_with?('MOVED')
           node = @router.assign_redirection_node(err.message)
-          try_redirection(node, pipeline, inner_index)
+          redirect_command(node, pipeline, inner_index)
         elsif err.message.start_with?('ASK')
           node = @router.assign_asking_node(err.message)
-          try_asking(node) ? try_redirection(node, pipeline, inner_index) : err
+          try_asking(node) ? redirect_command(node, pipeline, inner_index) : err
         else
           err
         end
       end
 
-      def try_redirection(node, pipeline, inner_index)
-        redirect_command(node, pipeline, inner_index)
-      rescue StandardError => e
-        e
-      end
-
       def redirect_command(node, pipeline, inner_index)
         method = pipeline.get_callee_method(inner_index)
         command = pipeline.get_command(inner_index)

data/lib/redis_client/cluster/router.rb

@@ -13,7 +13,6 @@ class RedisClient
   class Cluster
     class Router
       ZERO_CURSOR_FOR_SCAN = '0'
-      METHODS_FOR_BLOCKING_CMD = %i[blocking_call_v blocking_call].freeze
       TSF = ->(f, x) { f.nil? ? x : f.call(x) }.curry
 
       def initialize(config, concurrent_worker, pool: nil, **kwargs)
@@ -23,9 +22,9 @@ class RedisClient
         @concurrent_worker = concurrent_worker
         @pool = pool
         @client_kwargs = kwargs
-        @node = fetch_cluster_info(@config, @concurrent_worker, pool: @pool, **@client_kwargs)
+        @node = ::RedisClient::Cluster::Node.new(concurrent_worker, config: config, pool: pool, **kwargs)
+        update_cluster_info!
         @command = ::RedisClient::Cluster::Command.load(@node.replica_clients.shuffle, slow_command_timeout: config.slow_command_timeout)
-        @mutex = Mutex.new
         @command_builder = @config.command_builder
       end
 
@@ -66,77 +65,61 @@ class RedisClient
         raise if e.errors.any?(::RedisClient::CircuitBreaker::OpenCircuitError)
 
         update_cluster_info! if e.errors.values.any? do |err|
-          err.message.start_with?('CLUSTERDOWN Hash slot not served')
+          @node.owns_error?(err) && err.message.start_with?('CLUSTERDOWN Hash slot not served')
         end
 
         raise
       end
 
       # @see https://redis.io/docs/reference/cluster-spec/#redirection-and-resharding Redirection and resharding
-      def try_send(node, method, command, args, retry_count: 3, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if args.empty?
-          # prevent memory allocation for variable-length args
-          node.public_send(method, command, &block)
-        else
-          node.public_send(method, *args, command, &block)
+      def try_send(node, method, command, args, retry_count: 3, &block)
+        handle_redirection(node, retry_count: retry_count) do |on_node|
+          if args.empty?
+            # prevent memory allocation for variable-length args
+            on_node.public_send(method, command, &block)
+          else
+            on_node.public_send(method, *args, command, &block)
+          end
         end
-      rescue ::RedisClient::CircuitBreaker::OpenCircuitError
-        raise
-      rescue ::RedisClient::CommandError => e
-        raise if retry_count <= 0
+      end
 
-        if e.message.start_with?('MOVED')
-          node = assign_redirection_node(e.message)
-          retry_count -= 1
-          retry
-        elsif e.message.start_with?('ASK')
-          node = assign_asking_node(e.message)
-          node.call('ASKING')
-          retry_count -= 1
-          retry
-        elsif e.message.start_with?('CLUSTERDOWN Hash slot not served')
-          update_cluster_info!
-          retry_count -= 1
-          retry
-        else
-          raise
+      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block)
+        handle_redirection(node, retry_count: retry_count) do |on_node|
+          on_node.public_send(method, *args, **kwargs, &block)
         end
-      rescue ::RedisClient::ConnectionError => e
-        raise if METHODS_FOR_BLOCKING_CMD.include?(method) && e.is_a?(RedisClient::ReadTimeoutError)
-        raise if retry_count <= 0
-
-        update_cluster_info!
-        retry_count -= 1
-        retry
       end
 
-      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize
-        node.public_send(method, *args, **kwargs, &block)
+      def handle_redirection(node, retry_count:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        yield node
       rescue ::RedisClient::CircuitBreaker::OpenCircuitError
         raise
       rescue ::RedisClient::CommandError => e
-        raise if retry_count <= 0
+        raise unless ErrorIdentification.client_owns_error?(e, node)
 
         if e.message.start_with?('MOVED')
           node = assign_redirection_node(e.message)
           retry_count -= 1
-          retry
+          retry if retry_count >= 0
         elsif e.message.start_with?('ASK')
           node = assign_asking_node(e.message)
-          node.call('ASKING')
           retry_count -= 1
-          retry
+          if retry_count >= 0
+            node.call('ASKING')
+            retry
+          end
         elsif e.message.start_with?('CLUSTERDOWN Hash slot not served')
           update_cluster_info!
           retry_count -= 1
-          retry
-        else
-          raise
+          retry if retry_count >= 0
         end
-      rescue ::RedisClient::ConnectionError
-        raise if retry_count <= 0
+        raise
+      rescue ::RedisClient::ConnectionError => e
+        raise unless ErrorIdentification.client_owns_error?(e, node)
 
         update_cluster_info!
+
+        raise if retry_count <= 0
+
         retry_count -= 1
         retry
       end
@@ -170,21 +153,30 @@ class RedisClient
         find_node(node_key)
       end
 
-      def find_node_key(command, seed: nil)
-        key = @command.extract_first_key(command)
-        slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
-
-        if @command.should_send_to_primary?(command)
-          @node.find_node_key_of_primary(slot) || @node.any_primary_node_key(seed: seed)
+      def find_node_key_by_key(key, seed: nil, primary: false)
+        if key && !key.empty?
+          slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
+          primary ? @node.find_node_key_of_primary(slot) : @node.find_node_key_of_replica(slot)
         else
-          @node.find_node_key_of_replica(slot, seed: seed) || @node.any_replica_node_key(seed: seed)
+          primary ? @node.any_primary_node_key(seed: seed) : @node.any_replica_node_key(seed: seed)
         end
       end
 
+      def find_primary_node_by_slot(slot)
+        node_key = @node.find_node_key_of_primary(slot)
+        find_node(node_key)
+      end
+
+      def find_node_key(command, seed: nil)
+        key = @command.extract_first_key(command)
+        find_node_key_by_key(key, seed: seed, primary: @command.should_send_to_primary?(command))
+      end
+
       def find_primary_node_key(command)
         key = @command.extract_first_key(command)
-        slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
-        @node.find_node_key_of_primary(slot)
+        return nil unless key&.size&.> 0
+
+        find_node_key_by_key(key, primary: true)
       end
 
       def find_node(node_key, retry_count: 3)
@@ -306,34 +298,8 @@ class RedisClient
         end
       end
 
-      def fetch_cluster_info(config, concurrent_worker, pool: nil, **kwargs)
-        node_info_list = ::RedisClient::Cluster::Node.load_info(config.per_node_key, concurrent_worker, slow_command_timeout: config.slow_command_timeout, **kwargs)
-        node_addrs = node_info_list.map { |i| ::RedisClient::Cluster::NodeKey.hashify(i.node_key) }
-        config.update_node(node_addrs)
-        ::RedisClient::Cluster::Node.new(
-          config.per_node_key,
-          concurrent_worker,
-          node_info_list: node_info_list,
-          pool: pool,
-          with_replica: config.use_replica?,
-          replica_affinity: config.replica_affinity,
-          **kwargs
-        )
-      end
-
       def update_cluster_info!
-        return if @mutex.locked?
-
-        @mutex.synchronize do
-          begin
-            @node.each(&:close)
-          rescue ::RedisClient::Cluster::ErrorCollection
-            # ignore
-          end
-
-          @config = @original_config.dup if @connect_with_original_config
-          @node = fetch_cluster_info(@config, @concurrent_worker, pool: @pool, **@client_kwargs)
-        end
+        @node.reload!
       end
     end
   end