redis-cluster-client 0.7.4 → 0.7.6

data/lib/redis_client/cluster/transaction.rb

@@ -1,56 +1,194 @@
 # frozen_string_literal: true
 
 require 'redis_client'
+require 'redis_client/cluster/pipeline'
+require 'redis_client/cluster/key_slot_converter'
 
 class RedisClient
   class Cluster
     class Transaction
       ConsistencyError = Class.new(::RedisClient::Error)
 
-      def initialize(router, command_builder)
+      def initialize(router, command_builder, watch)
         @router = router
         @command_builder = command_builder
-        @node_key = nil
+        @watch = watch
+        @retryable = true
+        @pipeline = ::RedisClient::Pipeline.new(@command_builder)
+        @buffer = []
+        @node = nil
       end
 
-      def call(*command, **kwargs, &_)
+      def call(*command, **kwargs, &block)
         command = @command_builder.generate(command, kwargs)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_v(command, &block)
+        else
+          @buffer << -> { @pipeline.call_v(command, &block) }
+        end
       end
 
-      def call_v(command, &_)
+      def call_v(command, &block)
         command = @command_builder.generate(command)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_v(command, &block)
+        else
+          @buffer << -> { @pipeline.call_v(command, &block) }
+        end
       end
 
-      def call_once(*command, **kwargs, &_)
+      def call_once(*command, **kwargs, &block)
+        @retryable = false
         command = @command_builder.generate(command, kwargs)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_once_v(command, &block)
+        else
+          @buffer << -> { @pipeline.call_once_v(command, &block) }
+        end
       end
 
-      def call_once_v(command, &_)
+      def call_once_v(command, &block)
+        @retryable = false
         command = @command_builder.generate(command)
-        ensure_node_key(command)
+        if prepare(command)
+          @pipeline.call_once_v(command, &block)
+        else
+          @buffer << -> { @pipeline.call_once_v(command, &block) }
+        end
       end
 
-      def execute(watch: nil, &block)
-        yield self
-        raise ArgumentError, 'empty transaction' if @node_key.nil?
+      def execute
+        @buffer.each(&:call)
 
-        node = @router.find_node(@node_key)
-        @router.try_delegate(node, :multi, watch: watch, &block)
+        raise ArgumentError, 'empty transaction' if @pipeline._empty?
+        raise ConsistencyError, "couldn't determine the node: #{@pipeline._commands}" if @node.nil?
+        raise ConsistencyError, "unsafe watch: #{@watch.join(' ')}" unless safe_watch?
+
+        settle
       end
 
       private
 
-      def ensure_node_key(command)
+      def watch?
+        !@watch.nil? && !@watch.empty?
+      end
+
+      def safe_watch?
+        return true unless watch?
+        return false if @node.nil?
+
+        slots = @watch.map do |k|
+          return false if k.nil? || k.empty?
+
+          ::RedisClient::Cluster::KeySlotConverter.convert(k)
+        end
+
+        return false if slots.uniq.size != 1
+
+        @router.find_primary_node_by_slot(slots.first) == @node
+      end
+
+      def prepare(command)
+        return true unless @node.nil?
+
         node_key = @router.find_primary_node_key(command)
-        raise ConsistencyError, "Client couldn't determine the node to be executed the transaction by: #{command}" if node_key.nil?
+        return false if node_key.nil?
+
+        @node = @router.find_node(node_key)
+        @pipeline.call('WATCH', *@watch) if watch?
+        @pipeline.call('MULTI')
+        @buffer.each(&:call)
+        @buffer.clear
+        true
+      end
+
+      def settle
+        @pipeline.call('EXEC')
+        @pipeline.call('UNWATCH') if watch?
+        send_transaction(@node, redirect: true)
+      end
+
+      def send_transaction(client, redirect:)
+        case client
+        when ::RedisClient then send_pipeline(client, redirect: redirect)
+        when ::RedisClient::Pooled then client.with { |c| send_pipeline(c, redirect: redirect) }
+        else raise NotImplementedError, "#{client.class.name}#multi for cluster client"
+        end
+      end
+
+      def send_pipeline(client, redirect:)
+        replies = client.ensure_connected_cluster_scoped(retryable: @retryable) do |connection|
+          commands = @pipeline._commands
+          client.middlewares.call_pipelined(commands, client.config) do
+            connection.call_pipelined(commands, nil)
+          rescue ::RedisClient::CommandError => e
+            return handle_command_error!(commands, e) if redirect
 
-        @node_key ||= node_key
-        raise ConsistencyError, "The transaction should be done for single node: #{@node_key}, #{node_key}" if node_key != @node_key
+            raise
+          end
+        end
+
+        offset = watch? ? 2 : 1
+        coerce_results!(replies[-offset], offset)
+      end
+
+      def coerce_results!(results, offset)
+        results.each_with_index do |result, index|
+          if result.is_a?(::RedisClient::CommandError)
+            result._set_command(@pipeline._commands[index + offset])
+            raise result
+          end
+
+          next if @pipeline._blocks.nil?
+
+          block = @pipeline._blocks[index + offset]
+          next if block.nil?
+
+          results[index] = block.call(result)
+        end
+
+        results
+      end
+
+      def handle_command_error!(commands, err)
+        if err.message.start_with?('CROSSSLOT')
+          raise ConsistencyError, "#{err.message}: #{err.command}"
+        elsif err.message.start_with?('MOVED', 'ASK')
+          ensure_the_same_node!(commands)
+          handle_redirection(err)
+        else
+          raise err
+        end
+      end
+
+      def ensure_the_same_node!(commands)
+        commands.each do |command|
+          node_key = @router.find_primary_node_key(command)
+          next if node_key.nil?
+
+          node = @router.find_node(node_key)
+          next if @node == node
+
+          raise ConsistencyError, "the transaction should be executed to a slot in a node: #{commands}"
+        end
+      end
+
+      def handle_redirection(err)
+        if err.message.start_with?('MOVED')
+          node = @router.assign_redirection_node(err.message)
+          send_transaction(node, redirect: false)
+        elsif err.message.start_with?('ASK')
+          node = @router.assign_asking_node(err.message)
+          try_asking(node) ? send_transaction(node, redirect: false) : err
+        else
+          raise err
+        end
+      end
 
-        nil
+      def try_asking(node)
+        node.call('ASKING') == 'OK'
+      rescue StandardError
+        false
       end
     end
   end

data/lib/redis_client/cluster.rb

@@ -5,6 +5,7 @@ require 'redis_client/cluster/pipeline'
 require 'redis_client/cluster/pub_sub'
 require 'redis_client/cluster/router'
 require 'redis_client/cluster/transaction'
+require 'redis_client/cluster/pinning_node'
 
 class RedisClient
   class Cluster
@@ -89,14 +90,26 @@ class RedisClient
       pipeline.execute
     end
 
-    def multi(watch: nil, &block)
-      ::RedisClient::Cluster::Transaction.new(@router, @command_builder).execute(watch: watch, &block)
+    def multi(watch: nil)
+      transaction = ::RedisClient::Cluster::Transaction.new(@router, @command_builder, watch)
+      yield transaction
+      transaction.execute
     end
 
     def pubsub
       ::RedisClient::Cluster::PubSub.new(@router, @command_builder)
     end
 
+    # TODO: This isn't an official public interface yet. Don't use in your production environment.
+    # @see https://github.com/redis-rb/redis-cluster-client/issues/299
+    def with(key: nil, hashtag: nil, write: true, _retry_count: 0, &_)
+      key = process_with_arguments(key, hashtag)
+
+      node_key = @router.find_node_key_by_key(key, primary: write)
+      node = @router.find_node(node_key)
+      yield ::RedisClient::Cluster::PinningNode.new(node)
+    end
+
     def close
       @concurrent_worker.close
       @router.close
@@ -105,6 +118,19 @@ class RedisClient
 
     private
 
+    def process_with_arguments(key, hashtag) # rubocop:disable Metrics/CyclomaticComplexity
+      raise ArgumentError, 'Only one of key or hashtag may be provided' if key && hashtag
+
+      if hashtag
+        # The documentation says not to wrap your hashtag in {}, but people will probably
+        # do it anyway and it's easy for us to fix here.
+        key = hashtag&.match?(/^{.*}$/) ? hashtag : "{#{hashtag}}"
+      end
+      raise ArgumentError, 'One of key or hashtag must be provided' if key.nil? || key.empty?
+
+      key
+    end
+
     def method_missing(name, *args, **kwargs, &block)
       if @router.command_exists?(name)
         args.unshift(name)

data/lib/redis_client/cluster_config.rb

@@ -18,10 +18,13 @@ class RedisClient
     MERGE_CONFIG_KEYS = %i[ssl username password].freeze
     IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
     MAX_WORKERS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', 5))
+    # It's used with slow queries of fetching meta data like CLUSTER NODES, COMMAND and so on.
+    SLOW_COMMAND_TIMEOUT = Float(ENV.fetch('REDIS_CLIENT_SLOW_COMMAND_TIMEOUT', -1))
 
     InvalidClientConfigError = Class.new(::RedisClient::Error)
 
-    attr_reader :command_builder, :client_config, :replica_affinity
+    attr_reader :command_builder, :client_config, :replica_affinity, :slow_command_timeout,
+                :connect_with_original_config, :startup_nodes
 
     def initialize(
       nodes: DEFAULT_NODES,
@@ -29,36 +32,29 @@ class RedisClient
       replica_affinity: :random,
       fixed_hostname: '',
       concurrency: nil,
+      connect_with_original_config: false,
       client_implementation: ::RedisClient::Cluster, # for redis gem
+      slow_command_timeout: SLOW_COMMAND_TIMEOUT,
+      command_builder: ::RedisClient::CommandBuilder,
       **client_config
     )
 
       @replica = true & replica
       @replica_affinity = replica_affinity.to_s.to_sym
       @fixed_hostname = fixed_hostname.to_s
-      @node_configs = build_node_configs(nodes.dup)
-      client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
-      @command_builder = client_config.fetch(:command_builder, ::RedisClient::CommandBuilder)
-      @client_config = merge_generic_config(client_config, @node_configs)
+      @command_builder = command_builder
+      node_configs = build_node_configs(nodes.dup)
+      @client_config = merge_generic_config(client_config, node_configs)
+      # Keep tabs on the original startup nodes we were constructed with
+      @startup_nodes = build_startup_nodes(node_configs)
       @concurrency = merge_concurrency_option(concurrency)
+      @connect_with_original_config = connect_with_original_config
       @client_implementation = client_implementation
-      @mutex = Mutex.new
-    end
-
-    def dup
-      self.class.new(
-        nodes: @node_configs,
-        replica: @replica,
-        replica_affinity: @replica_affinity,
-        fixed_hostname: @fixed_hostname,
-        concurrency: @concurrency,
-        client_implementation: @client_implementation,
-        **@client_config
-      )
+      @slow_command_timeout = slow_command_timeout
     end
 
     def inspect
-      "#<#{self.class.name} #{per_node_key.values}>"
+      "#<#{self.class.name} #{startup_nodes.values}>"
     end
 
     def read_timeout
@@ -78,29 +74,14 @@ class RedisClient
       @client_implementation.new(self, concurrency: @concurrency, **kwargs)
     end
 
-    def per_node_key
-      @node_configs.to_h do |config|
-        node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
-        config = @client_config.merge(config)
-        config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty?
-        [node_key, config]
-      end
-    end
-
     def use_replica?
       @replica
     end
 
-    def update_node(addrs)
-      return if @mutex.locked?
-
-      @mutex.synchronize { @node_configs = build_node_configs(addrs) }
-    end
-
-    def add_node(host, port)
-      return if @mutex.locked?
-
-      @mutex.synchronize { @node_configs << { host: host, port: port } }
+    def client_config_for_node(node_key)
+      config = ::RedisClient::Cluster::NodeKey.hashify(node_key)
+      config[:port] = ensure_integer(config[:port])
+      augment_client_config(config)
     end
 
     private
@@ -168,11 +149,23 @@ class RedisClient
     end
 
     def merge_generic_config(client_config, node_configs)
-      return client_config if node_configs.empty?
+      cfg = node_configs.first || {}
+      client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
+                   .merge(cfg.slice(*MERGE_CONFIG_KEYS))
+    end
+
+    def build_startup_nodes(configs)
+      configs.to_h do |config|
+        node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
+        config = augment_client_config(config)
+        [node_key, config]
+      end
+    end
 
-      cfg = node_configs.first
-      MERGE_CONFIG_KEYS.each { |k| client_config[k] = cfg[k] if cfg.key?(k) }
-      client_config
+    def augment_client_config(config)
+      config = @client_config.merge(config)
+      config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty?
+      config
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.6
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-06 00:00:00.000000000 Z
+date: 2024-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -38,16 +38,18 @@ files:
 - lib/redis_client/cluster/concurrent_worker/none.rb
 - lib/redis_client/cluster/concurrent_worker/on_demand.rb
 - lib/redis_client/cluster/concurrent_worker/pooled.rb
+- lib/redis_client/cluster/error_identification.rb
 - lib/redis_client/cluster/errors.rb
 - lib/redis_client/cluster/key_slot_converter.rb
 - lib/redis_client/cluster/node.rb
+- lib/redis_client/cluster/node/base_topology.rb
 - lib/redis_client/cluster/node/latency_replica.rb
 - lib/redis_client/cluster/node/primary_only.rb
 - lib/redis_client/cluster/node/random_replica.rb
 - lib/redis_client/cluster/node/random_replica_or_primary.rb
-- lib/redis_client/cluster/node/replica_mixin.rb
 - lib/redis_client/cluster/node_key.rb
 - lib/redis_client/cluster/normalized_cmd_name.rb
+- lib/redis_client/cluster/pinning_node.rb
 - lib/redis_client/cluster/pipeline.rb
 - lib/redis_client/cluster/pub_sub.rb
 - lib/redis_client/cluster/router.rb
@@ -75,7 +77,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: A Redis cluster client for Ruby

data/lib/redis_client/cluster/node/replica_mixin.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-class RedisClient
-  class Cluster
-    class Node
-      module ReplicaMixin
-        attr_reader :clients, :primary_clients
-
-        EMPTY_ARRAY = [].freeze
-
-        def initialize(replications, options, pool, _concurrent_worker, **kwargs)
-          @replications = replications
-          @primary_node_keys = @replications.keys.sort
-          @replica_node_keys = @replications.values.flatten.sort
-          @clients = build_clients(@primary_node_keys, options, pool, **kwargs)
-          @primary_clients = @clients.select { |k, _| @primary_node_keys.include?(k) }
-        end
-
-        def any_primary_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @primary_node_keys.sample(random: random)
-        end
-
-        private
-
-        def build_clients(primary_node_keys, options, pool, **kwargs)
-          options.to_h do |node_key, option|
-            option = option.merge(kwargs.reject { |k, _| ::RedisClient::Cluster::Node::IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
-            config = ::RedisClient::Cluster::Node::Config.new(scale_read: !primary_node_keys.include?(node_key), **option)
-            client = pool.nil? ? config.new_client : config.new_pool(**pool)
-            [node_key, client]
-          end
-        end
-      end
-    end
-  end
-end