redis-cluster-client 0.7.5 → 0.7.7

data/lib/redis_client/cluster/router.rb

@@ -13,7 +13,6 @@ class RedisClient
   class Cluster
     class Router
       ZERO_CURSOR_FOR_SCAN = '0'
-      METHODS_FOR_BLOCKING_CMD = %i[blocking_call_v blocking_call].freeze
       TSF = ->(f, x) { f.nil? ? x : f.call(x) }.curry
 
       def initialize(config, concurrent_worker, pool: nil, **kwargs)
@@ -23,9 +22,9 @@ class RedisClient
         @concurrent_worker = concurrent_worker
         @pool = pool
         @client_kwargs = kwargs
-        @node = fetch_cluster_info(@config, @concurrent_worker, pool: @pool, **@client_kwargs)
+        @node = ::RedisClient::Cluster::Node.new(concurrent_worker, config: config, pool: pool, **kwargs)
+        update_cluster_info!
         @command = ::RedisClient::Cluster::Command.load(@node.replica_clients.shuffle, slow_command_timeout: config.slow_command_timeout)
-        @mutex = Mutex.new
         @command_builder = @config.command_builder
       end
 
@@ -66,77 +65,61 @@ class RedisClient
         raise if e.errors.any?(::RedisClient::CircuitBreaker::OpenCircuitError)
 
         update_cluster_info! if e.errors.values.any? do |err|
-          err.message.start_with?('CLUSTERDOWN Hash slot not served')
+          @node.owns_error?(err) && err.message.start_with?('CLUSTERDOWN Hash slot not served')
         end
 
         raise
       end
 
       # @see https://redis.io/docs/reference/cluster-spec/#redirection-and-resharding Redirection and resharding
-      def try_send(node, method, command, args, retry_count: 3, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if args.empty?
-          # prevent memory allocation for variable-length args
-          node.public_send(method, command, &block)
-        else
-          node.public_send(method, *args, command, &block)
+      def try_send(node, method, command, args, retry_count: 3, &block)
+        handle_redirection(node, retry_count: retry_count) do |on_node|
+          if args.empty?
+            # prevent memory allocation for variable-length args
+            on_node.public_send(method, command, &block)
+          else
+            on_node.public_send(method, *args, command, &block)
+          end
         end
-      rescue ::RedisClient::CircuitBreaker::OpenCircuitError
-        raise
-      rescue ::RedisClient::CommandError => e
-        raise if retry_count <= 0
+      end
 
-        if e.message.start_with?('MOVED')
-          node = assign_redirection_node(e.message)
-          retry_count -= 1
-          retry
-        elsif e.message.start_with?('ASK')
-          node = assign_asking_node(e.message)
-          node.call('ASKING')
-          retry_count -= 1
-          retry
-        elsif e.message.start_with?('CLUSTERDOWN Hash slot not served')
-          update_cluster_info!
-          retry_count -= 1
-          retry
-        else
-          raise
+      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block)
+        handle_redirection(node, retry_count: retry_count) do |on_node|
+          on_node.public_send(method, *args, **kwargs, &block)
         end
-      rescue ::RedisClient::ConnectionError => e
-        raise if METHODS_FOR_BLOCKING_CMD.include?(method) && e.is_a?(RedisClient::ReadTimeoutError)
-        raise if retry_count <= 0
-
-        update_cluster_info!
-        retry_count -= 1
-        retry
       end
 
-      def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize
-        node.public_send(method, *args, **kwargs, &block)
+      def handle_redirection(node, retry_count:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        yield node
       rescue ::RedisClient::CircuitBreaker::OpenCircuitError
         raise
       rescue ::RedisClient::CommandError => e
-        raise if retry_count <= 0
+        raise unless ErrorIdentification.client_owns_error?(e, node)
 
         if e.message.start_with?('MOVED')
           node = assign_redirection_node(e.message)
           retry_count -= 1
-          retry
+          retry if retry_count >= 0
         elsif e.message.start_with?('ASK')
           node = assign_asking_node(e.message)
-          node.call('ASKING')
           retry_count -= 1
-          retry
+          if retry_count >= 0
+            node.call('ASKING')
+            retry
+          end
         elsif e.message.start_with?('CLUSTERDOWN Hash slot not served')
           update_cluster_info!
           retry_count -= 1
-          retry
-        else
-          raise
+          retry if retry_count >= 0
         end
-      rescue ::RedisClient::ConnectionError
-        raise if retry_count <= 0
+        raise
+      rescue ::RedisClient::ConnectionError => e
+        raise unless ErrorIdentification.client_owns_error?(e, node)
 
         update_cluster_info!
+
+        raise if retry_count <= 0
+
         retry_count -= 1
         retry
       end
@@ -170,21 +153,30 @@ class RedisClient
         find_node(node_key)
       end
 
-      def find_node_key(command, seed: nil)
-        key = @command.extract_first_key(command)
-        slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
-
-        if @command.should_send_to_primary?(command)
-          @node.find_node_key_of_primary(slot) || @node.any_primary_node_key(seed: seed)
+      def find_node_key_by_key(key, seed: nil, primary: false)
+        if key && !key.empty?
+          slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
+          primary ? @node.find_node_key_of_primary(slot) : @node.find_node_key_of_replica(slot)
         else
-          @node.find_node_key_of_replica(slot, seed: seed) || @node.any_replica_node_key(seed: seed)
+          primary ? @node.any_primary_node_key(seed: seed) : @node.any_replica_node_key(seed: seed)
         end
       end
 
+      def find_primary_node_by_slot(slot)
+        node_key = @node.find_node_key_of_primary(slot)
+        find_node(node_key)
+      end
+
+      def find_node_key(command, seed: nil)
+        key = @command.extract_first_key(command)
+        find_node_key_by_key(key, seed: seed, primary: @command.should_send_to_primary?(command))
+      end
+
       def find_primary_node_key(command)
         key = @command.extract_first_key(command)
-        slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
-        @node.find_node_key_of_primary(slot)
+        return nil unless key&.size&.> 0
+
+        find_node_key_by_key(key, primary: true)
       end
 
       def find_node(node_key, retry_count: 3)
@@ -306,34 +298,8 @@ class RedisClient
         end
       end
 
-      def fetch_cluster_info(config, concurrent_worker, pool: nil, **kwargs)
-        node_info_list = ::RedisClient::Cluster::Node.load_info(config.per_node_key, concurrent_worker, slow_command_timeout: config.slow_command_timeout, **kwargs)
-        node_addrs = node_info_list.map { |i| ::RedisClient::Cluster::NodeKey.hashify(i.node_key) }
-        config.update_node(node_addrs)
-        ::RedisClient::Cluster::Node.new(
-          config.per_node_key,
-          concurrent_worker,
-          node_info_list: node_info_list,
-          pool: pool,
-          with_replica: config.use_replica?,
-          replica_affinity: config.replica_affinity,
-          **kwargs
-        )
-      end
-
       def update_cluster_info!
-        return if @mutex.locked?
-
-        @mutex.synchronize do
-          begin
-            @node.each(&:close)
-          rescue ::RedisClient::Cluster::ErrorCollection
-            # ignore
-          end
-
-          @config = @original_config.dup if @connect_with_original_config
-          @node = fetch_cluster_info(@config, @concurrent_worker, pool: @pool, **@client_kwargs)
-        end
+        @node.reload!
       end
     end
   end

data/lib/redis_client/cluster/transaction.rb

@@ -1,56 +1,182 @@
 # frozen_string_literal: true
 
 require 'redis_client'
+require 'redis_client/cluster/pipeline'
+require 'redis_client/cluster/node_key'
 
 class RedisClient
   class Cluster
     class Transaction
       ConsistencyError = Class.new(::RedisClient::Error)
 
-      def initialize(router, command_builder)
+      def initialize(router, command_builder, node = nil)
         @router = router
         @command_builder = command_builder
-        @node_key = nil
+        @retryable = true
+        @pipeline = ::RedisClient::Pipeline.new(@command_builder)
+        @pending_commands = []
+        @node = node
+        prepare_tx unless @node.nil?
       end
 
-      def call(*command, **kwargs, &_)
+      def call(*command, **kwargs, &block)
         command = @command_builder.generate(command, kwargs)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_v(command, &block)
+        else
+          defer { @pipeline.call_v(command, &block) }
+        end
       end
 
-      def call_v(command, &_)
+      def call_v(command, &block)
         command = @command_builder.generate(command)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_v(command, &block)
+        else
+          defer { @pipeline.call_v(command, &block) }
+        end
       end
 
-      def call_once(*command, **kwargs, &_)
+      def call_once(*command, **kwargs, &block)
+        @retryable = false
         command = @command_builder.generate(command, kwargs)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_once_v(command, &block)
+        else
+          defer { @pipeline.call_once_v(command, &block) }
+        end
       end
 
-      def call_once_v(command, &_)
+      def call_once_v(command, &block)
+        @retryable = false
         command = @command_builder.generate(command)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_once_v(command, &block)
+        else
+          defer { @pipeline.call_once_v(command, &block) }
+        end
       end
 
-      def execute(watch: nil, &block)
-        yield self
-        raise ArgumentError, 'empty transaction' if @node_key.nil?
+      def execute
+        @pending_commands.each(&:call)
 
-        node = @router.find_node(@node_key)
-        @router.try_delegate(node, :multi, watch: watch, &block)
+        raise ArgumentError, 'empty transaction' if @pipeline._empty?
+        raise ConsistencyError, "couldn't determine the node: #{@pipeline._commands}" if @node.nil?
+
+        settle
       end
 
       private
 
-      def ensure_node_key(command)
+      def defer(&block)
+        @pending_commands << block
+        nil
+      end
+
+      def prepare(command)
+        return true unless @node.nil?
+
         node_key = @router.find_primary_node_key(command)
-        raise ConsistencyError, "Client couldn't determine the node to be executed the transaction by: #{command}" if node_key.nil?
+        return false if node_key.nil?
 
-        @node_key ||= node_key
-        raise ConsistencyError, "The transaction should be done for single node: #{@node_key}, #{node_key}" if node_key != @node_key
+        @node = @router.find_node(node_key)
+        prepare_tx
+        true
+      end
 
-        nil
+      def prepare_tx
+        @pipeline.call('MULTI')
+        @pending_commands.each(&:call)
+        @pending_commands.clear
+      end
+
+      def settle
+        @pipeline.call('EXEC')
+        send_transaction(@node, redirect: true)
+      end
+
+      def send_transaction(client, redirect:)
+        case client
+        when ::RedisClient then send_pipeline(client, redirect: redirect)
+        when ::RedisClient::Pooled then client.with { |c| send_pipeline(c, redirect: redirect) }
+        else raise NotImplementedError, "#{client.class.name}#multi for cluster client"
+        end
+      end
+
+      def send_pipeline(client, redirect:)
+        replies = client.ensure_connected_cluster_scoped(retryable: @retryable) do |connection|
+          commands = @pipeline._commands
+          client.middlewares.call_pipelined(commands, client.config) do
+            connection.call_pipelined(commands, nil)
+          rescue ::RedisClient::CommandError => e
+            return handle_command_error!(commands, e) if redirect
+
+            raise
+          end
+        end
+
+        return if replies.last.nil?
+
+        coerce_results!(replies.last)
+      end
+
+      def coerce_results!(results, offset: 1)
+        results.each_with_index do |result, index|
+          if result.is_a?(::RedisClient::CommandError)
+            result._set_command(@pipeline._commands[index + offset])
+            raise result
+          end
+
+          next if @pipeline._blocks.nil?
+
+          block = @pipeline._blocks[index + offset]
+          next if block.nil?
+
+          results[index] = block.call(result)
+        end
+
+        results
+      end
+
+      def handle_command_error!(commands, err)
+        if err.message.start_with?('CROSSSLOT')
+          raise ConsistencyError, "#{err.message}: #{err.command}"
+        elsif err.message.start_with?('MOVED', 'ASK')
+          ensure_the_same_node!(commands)
+          handle_redirection(err)
+        else
+          raise err
+        end
+      end
+
+      def ensure_the_same_node!(commands)
+        expected_node_key = NodeKey.build_from_client(@node)
+
+        commands.each do |command|
+          node_key = @router.find_primary_node_key(command)
+          next if node_key.nil?
+          next if node_key == expected_node_key
+
+          raise ConsistencyError, "the transaction should be executed to a slot in a node: #{commands}"
+        end
+      end
+
+      def handle_redirection(err)
+        if err.message.start_with?('MOVED')
+          node = @router.assign_redirection_node(err.message)
+          send_transaction(node, redirect: false)
+        elsif err.message.start_with?('ASK')
+          node = @router.assign_asking_node(err.message)
+          try_asking(node) ? send_transaction(node, redirect: false) : err
+        else
+          raise err
+        end
+      end
+
+      def try_asking(node)
+        node.call('ASKING') == 'OK'
+      rescue StandardError
+        false
       end
     end
   end

data/lib/redis_client/cluster.rb

@@ -5,6 +5,8 @@ require 'redis_client/cluster/pipeline'
 require 'redis_client/cluster/pub_sub'
 require 'redis_client/cluster/router'
 require 'redis_client/cluster/transaction'
+require 'redis_client/cluster/pinning_node'
+require 'redis_client/cluster/optimistic_locking'
 
 class RedisClient
   class Cluster
@@ -89,14 +91,34 @@ class RedisClient
       pipeline.execute
     end
 
-    def multi(watch: nil, &block)
-      ::RedisClient::Cluster::Transaction.new(@router, @command_builder).execute(watch: watch, &block)
+    def multi(watch: nil)
+      if watch.nil? || watch.empty?
+        transaction = ::RedisClient::Cluster::Transaction.new(@router, @command_builder)
+        yield transaction
+        transaction.execute
+      else
+        locking = ::RedisClient::Cluster::OptimisticLocking.new(watch, @router)
+        locking.watch do |c|
+          transaction = ::RedisClient::Cluster::Transaction.new(@router, @command_builder, c)
+          yield transaction
+          transaction.execute
+        end
+      end
     end
 
     def pubsub
       ::RedisClient::Cluster::PubSub.new(@router, @command_builder)
     end
 
+    # TODO: This isn't an official public interface yet. Don't use in your production environment.
+    # @see https://github.com/redis-rb/redis-cluster-client/issues/299
+    def with(key: nil, hashtag: nil, write: true)
+      key = process_with_arguments(key, hashtag)
+      node_key = @router.find_node_key_by_key(key, primary: write)
+      node = @router.find_node(node_key)
+      node.with { |c| yield ::RedisClient::Cluster::PinningNode.new(c) }
+    end
+
     def close
       @concurrent_worker.close
       @router.close
@@ -105,6 +127,19 @@ class RedisClient
 
     private
 
+    def process_with_arguments(key, hashtag) # rubocop:disable Metrics/CyclomaticComplexity
+      raise ArgumentError, 'Only one of key or hashtag may be provided' if key && hashtag
+
+      if hashtag
+        # The documentation says not to wrap your hashtag in {}, but people will probably
+        # do it anyway and it's easy for us to fix here.
+        key = hashtag&.match?(/^{.*}$/) ? hashtag : "{#{hashtag}}"
+      end
+      raise ArgumentError, 'One of key or hashtag must be provided' if key.nil? || key.empty?
+
+      key
+    end
+
     def method_missing(name, *args, **kwargs, &block)
       if @router.command_exists?(name)
         args.unshift(name)

data/lib/redis_client/cluster_config.rb

@@ -23,9 +23,10 @@ class RedisClient
 
     InvalidClientConfigError = Class.new(::RedisClient::Error)
 
-    attr_reader :command_builder, :client_config, :replica_affinity, :slow_command_timeout, :connect_with_original_config
+    attr_reader :command_builder, :client_config, :replica_affinity, :slow_command_timeout,
+                :connect_with_original_config, :startup_nodes
 
-    def initialize( # rubocop:disable Metrics/AbcSize
+    def initialize(
       nodes: DEFAULT_NODES,
       replica: false,
       replica_affinity: :random,
@@ -34,39 +35,26 @@ class RedisClient
       connect_with_original_config: false,
       client_implementation: ::RedisClient::Cluster, # for redis gem
       slow_command_timeout: SLOW_COMMAND_TIMEOUT,
+      command_builder: ::RedisClient::CommandBuilder,
       **client_config
     )
 
       @replica = true & replica
       @replica_affinity = replica_affinity.to_s.to_sym
       @fixed_hostname = fixed_hostname.to_s
-      @node_configs = build_node_configs(nodes.dup)
-      client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
-      @command_builder = client_config.fetch(:command_builder, ::RedisClient::CommandBuilder)
-      @client_config = merge_generic_config(client_config, @node_configs)
+      @command_builder = command_builder
+      node_configs = build_node_configs(nodes.dup)
+      @client_config = merge_generic_config(client_config, node_configs)
+      # Keep tabs on the original startup nodes we were constructed with
+      @startup_nodes = build_startup_nodes(node_configs)
       @concurrency = merge_concurrency_option(concurrency)
       @connect_with_original_config = connect_with_original_config
       @client_implementation = client_implementation
       @slow_command_timeout = slow_command_timeout
-      @mutex = Mutex.new
-    end
-
-    def dup
-      self.class.new(
-        nodes: @node_configs,
-        replica: @replica,
-        replica_affinity: @replica_affinity,
-        fixed_hostname: @fixed_hostname,
-        concurrency: @concurrency,
-        connect_with_original_config: @connect_with_original_config,
-        client_implementation: @client_implementation,
-        slow_command_timeout: @slow_command_timeout,
-        **@client_config
-      )
     end
 
     def inspect
-      "#<#{self.class.name} #{per_node_key.values}>"
+      "#<#{self.class.name} #{startup_nodes.values}>"
     end
 
     def read_timeout
@@ -86,29 +74,14 @@ class RedisClient
       @client_implementation.new(self, concurrency: @concurrency, **kwargs)
     end
 
-    def per_node_key
-      @node_configs.to_h do |config|
-        node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
-        config = @client_config.merge(config)
-        config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty?
-        [node_key, config]
-      end
-    end
-
     def use_replica?
       @replica
     end
 
-    def update_node(addrs)
-      return if @mutex.locked?
-
-      @mutex.synchronize { @node_configs = build_node_configs(addrs) }
-    end
-
-    def add_node(host, port)
-      return if @mutex.locked?
-
-      @mutex.synchronize { @node_configs << { host: host, port: port } }
+    def client_config_for_node(node_key)
+      config = ::RedisClient::Cluster::NodeKey.hashify(node_key)
+      config[:port] = ensure_integer(config[:port])
+      augment_client_config(config)
     end
 
     private
@@ -176,11 +149,23 @@ class RedisClient
     end
 
     def merge_generic_config(client_config, node_configs)
-      return client_config if node_configs.empty?
+      cfg = node_configs.first || {}
+      client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
+                   .merge(cfg.slice(*MERGE_CONFIG_KEYS))
+    end
+
+    def build_startup_nodes(configs)
+      configs.to_h do |config|
+        node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
+        config = augment_client_config(config)
+        [node_key, config]
+      end
+    end
 
-      cfg = node_configs.first
-      MERGE_CONFIG_KEYS.each { |k| client_config[k] = cfg[k] if cfg.key?(k) }
-      client_config
+    def augment_client_config(config)
+      config = @client_config.merge(config)
+      config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty?
+      config
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.7
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-20 00:00:00.000000000 Z
+date: 2024-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -38,16 +38,19 @@ files:
 - lib/redis_client/cluster/concurrent_worker/none.rb
 - lib/redis_client/cluster/concurrent_worker/on_demand.rb
 - lib/redis_client/cluster/concurrent_worker/pooled.rb
+- lib/redis_client/cluster/error_identification.rb
 - lib/redis_client/cluster/errors.rb
 - lib/redis_client/cluster/key_slot_converter.rb
 - lib/redis_client/cluster/node.rb
+- lib/redis_client/cluster/node/base_topology.rb
 - lib/redis_client/cluster/node/latency_replica.rb
 - lib/redis_client/cluster/node/primary_only.rb
 - lib/redis_client/cluster/node/random_replica.rb
 - lib/redis_client/cluster/node/random_replica_or_primary.rb
-- lib/redis_client/cluster/node/replica_mixin.rb
 - lib/redis_client/cluster/node_key.rb
 - lib/redis_client/cluster/normalized_cmd_name.rb
+- lib/redis_client/cluster/optimistic_locking.rb
+- lib/redis_client/cluster/pinning_node.rb
 - lib/redis_client/cluster/pipeline.rb
 - lib/redis_client/cluster/pub_sub.rb
 - lib/redis_client/cluster/router.rb
@@ -75,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: A Redis cluster client for Ruby

data/lib/redis_client/cluster/node/replica_mixin.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-class RedisClient
-  class Cluster
-    class Node
-      module ReplicaMixin
-        attr_reader :clients, :primary_clients
-
-        EMPTY_ARRAY = [].freeze
-
-        def initialize(replications, options, pool, _concurrent_worker, **kwargs)
-          @replications = replications
-          @primary_node_keys = @replications.keys.sort
-          @replica_node_keys = @replications.values.flatten.sort
-          @clients = build_clients(@primary_node_keys, options, pool, **kwargs)
-          @primary_clients = @clients.select { |k, _| @primary_node_keys.include?(k) }
-        end
-
-        def any_primary_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @primary_node_keys.sample(random: random)
-        end
-
-        private
-
-        def build_clients(primary_node_keys, options, pool, **kwargs)
-          options.to_h do |node_key, option|
-            option = option.merge(kwargs.reject { |k, _| ::RedisClient::Cluster::Node::IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
-            config = ::RedisClient::Cluster::Node::Config.new(scale_read: !primary_node_keys.include?(node_key), **option)
-            client = pool.nil? ? config.new_client : config.new_pool(**pool)
-            [node_key, client]
-          end
-        end
-      end
-    end
-  end
-end