redis-cluster-client 0.0.3 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a4668f3eaba9c3053d1464bdf1dc85524e2ba43fdbb2ffbbadc46fb4759d5e6
-  data.tar.gz: eba5280f698fe559a69cdeba28cef2c0fff87023fa78e7d098042d9cc6c036ac
+  metadata.gz: fde84481c2ca9680670f252a002391a34ae793148a21a8e02086d199412a53ab
+  data.tar.gz: 5af188432f9ae5e76c13d0dd174c25cde4b98d1de49efeb97e2a809bd111a9a6
 SHA512:
-  metadata.gz: 989fd1b82d429ae6450a8c9e1555fc3635c1592c4d68744ad33df9cb26f5d222666fbeb077af99757c6305956936f44b8e38aef405c1a275c10e501559337934
-  data.tar.gz: c0e88ca74a4d0ff70116969ea0722d1659858587c1cfe8a404edf41658c0b9a87e0f3651db02fe88ad6d099b2948814fcc9415aa793394fb4e75ebc5eef0acec
+  metadata.gz: f26dc7bb447ee70742efc90344758ad3058cd0d1506ad8576637b7e9ad296a806d7e15e3b824cead0fb2313140dea00f6f579a15803399c16257ac347c45085f
+  data.tar.gz: 06f43e223a53d9ada0803f7292bc1ca58505fb3c341fd7643eb6cf0eff73bebe4ecd2022b0a1b59af81dda879a9d8a9b078107181ff45037773382dada321597

data/lib/redis_client/cluster/node.rb

@@ -79,6 +79,7 @@ class RedisClient
         @slots = build_slot_node_mappings(node_info)
         @replications = build_replication_mappings(node_info)
         @clients = build_clients(options, pool: pool, **kwargs)
+        @mutex = Mutex.new
       end
 
       def inspect
@@ -101,6 +102,12 @@ class RedisClient
         @clients.filter_map { |k, _| primary?(k) ? k : nil }.sort
       end
 
+      def replica_node_keys
+        return primary_node_keys if replica_disabled?
+
+        @clients.filter_map { |k, _| replica?(k) ? k : nil }.sort
+      end
+
       def find_by(node_key)
         @clients.fetch(node_key)
       rescue KeyError
@@ -122,19 +129,17 @@ class RedisClient
       def call_replica(method, *command, **kwargs, &block)
         return call_primary(method, *command, **kwargs, &block) if replica_disabled?
 
+        replica_node_keys = @replications.values.map(&:sample)
         try_map do |node_key, client|
-          next if primary?(node_key)
+          next if primary?(node_key) || !replica_node_keys.include?(node_key)
 
           client.send(method, *command, **kwargs, &block)
         end.values
       end
 
       def scale_reading_clients
-        clients = @clients.select do |node_key, _|
-          replica_disabled? ? primary?(node_key) : replica?(node_key)
-        end
-
-        clients.values.sort_by do |client|
+        keys = replica_disabled? ? @replications.keys : @replications.values.map(&:first)
+        @clients.select { |k, _| keys.include?(k) }.values.sort_by do |client|
           ::RedisClient::Cluster::NodeKey.build_from_host_port(client.config.host, client.config.port)
         end
       end
@@ -163,7 +168,13 @@ class RedisClient
       end
 
       def update_slot(slot, node_key)
-        @slots[slot] = node_key
+        @mutex.synchronize { @slots[slot] = node_key }
+      end
+
+      def replicated?(primary_node_key, replica_node_key)
+        return false if @replications.nil? || @replications.size.zero?
+
+        @replications.fetch(primary_node_key).include?(replica_node_key)
       end
 
       private
@@ -177,7 +188,9 @@ class RedisClient
       end
 
       def replica?(node_key)
-        !(@replications.nil? || @replications.size.zero?) && @replications[node_key].size.zero?
+        return false if @replications.nil? || @replications.size.zero?
+
+        !@replications.key?(node_key)
       end
 
       def build_slot_node_mappings(node_info)
@@ -191,12 +204,12 @@ class RedisClient
         slots
       end
 
-      def build_replication_mappings(node_info)
+      def build_replication_mappings(node_info) # rubocop:disable Metrics/AbcSize
         dict = node_info.to_h { |info| [info[:id], info] }
         node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
           primary_info = dict[info[:primary_id]]
           acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
-          acc[info[:node_key]]
+          acc[info[:node_key]] if info[:role] == 'master' # for the primary which have no replicas
         end
       end
 
@@ -217,15 +230,17 @@ class RedisClient
       def try_map # rubocop:disable Metrics/MethodLength
         errors = {}
         results = {}
-
-        @clients.each do |node_key, client|
-          reply = yield(node_key, client)
-          results[node_key] = reply unless reply.nil?
-        rescue ::RedisClient::CommandError => e
-          errors[node_key] = e
-          next
+        threads = @clients.map do |k, v|
+          Thread.new(k, v) do |node_key, client|
+            Thread.pass
+            reply = yield(node_key, client)
+            results[node_key] = reply unless reply.nil?
+          rescue ::RedisClient::CommandError => e
+            errors[node_key] = e
+          end
         end
 
+        threads.each(&:join)
         return results if errors.empty?
 
         raise ::RedisClient::Cluster::CommandErrorCollection, errors

data/lib/redis_client/cluster.rb

@@ -15,7 +15,6 @@ class RedisClient
       def initialize(client)
         @client = client
         @grouped = Hash.new([].freeze)
-        @replies = []
         @size = 0
       end
 
@@ -41,27 +40,31 @@ class RedisClient
         @size.zero?
       end
 
-      # TODO: use concurrency
+      # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37
       def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-        @grouped.each do |node_key, rows|
-          node_key = node_key.nil? ? @client.instance_variable_get(:@node).primary_node_keys.sample : node_key
-          replies = @client.send(:find_node, node_key).pipelined do |pipeline|
-            rows.each do |row|
-              case row[1]
-              when :call then pipeline.call(*row[2], **row[3])
-              when :call_once then pipeline.call_once(*row[2], **row[3])
-              when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
-              else raise NotImplementedError, row[1]
+        all_replies = Array.new(@size)
+        threads = @grouped.map do |k, v|
+          Thread.new(@client, k, v) do |client, node_key, rows|
+            Thread.pass
+            replies = client.send(:find_node, node_key).pipelined do |pipeline|
+              rows.each do |row|
+                case row[1]
+                when :call then pipeline.call(*row[2], **row[3])
+                when :call_once then pipeline.call_once(*row[2], **row[3])
+                when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
+                else raise NotImplementedError, row[1]
+                end
               end
             end
-          end
 
-          raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
+            raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
 
-          rows.each_with_index { |row, idx| @replies[row.first] = replies[idx] }
+            rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
+          end
         end
 
-        @replies
+        threads.each(&:join)
+        all_replies
       end
     end
 
@@ -103,6 +106,7 @@ class RedisClient
       @client_kwargs = kwargs
       @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
       @command = ::RedisClient::Cluster::Command.load(@node)
+      @mutex = Mutex.new
     end
 
     def inspect
@@ -178,11 +182,11 @@ class RedisClient
     def send_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
       cmd = command.first.to_s.downcase
       case cmd
-      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
+      when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save', 'ping'
         @node.call_all(method, *command, **kwargs, &block).first
       when 'flushall', 'flushdb'
         @node.call_primary(method, *command, **kwargs, &block).first
-      when 'wait'     then @node.call_primary(method, *command, **kwargs, &block).sum
+      when 'wait'     then send_wait_command(method, *command, **kwargs, &block)
       when 'keys'     then @node.call_replica(method, *command, **kwargs, &block).flatten.sort
       when 'dbsize'   then @node.call_replica(method, *command, **kwargs, &block).sum
       when 'scan'     then _scan(*command, **kwargs)
@@ -202,6 +206,19 @@ class RedisClient
         node = assign_node(*command)
         try_send(node, method, *command, **kwargs, &block)
       end
+    rescue RedisClient::Cluster::CommandErrorCollection => e
+      update_cluster_info! if e.errors.values.map(&:class).any?(::RedisClient::ConnectionError)
+      raise
+    end
+
+    def send_wait_command(method, *command, retry_count: 3, **kwargs, &block)
+      @node.call_primary(method, *command, **kwargs, &block).sum
+    rescue RedisClient::Cluster::CommandErrorCollection => e
+      raise if retry_count <= 0 || e.errors.values.map(&:message).grep(/ERR WAIT cannot be used with replica instances/).empty?
+
+      update_cluster_info!
+      retry_count -= 1
+      retry
     end
 
     def send_config_command(method, *command, **kwargs, &block)
@@ -263,18 +280,16 @@ class RedisClient
 
     # @see https://redis.io/topics/cluster-spec#redirection-and-resharding
     #   Redirection and resharding
-    def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       node.send(method, *args, **kwargs, &block)
     rescue ::RedisClient::CommandError => e
-      if e.message.start_with?(REPLY_MOVED)
-        raise if retry_count <= 0
+      raise if retry_count <= 0
 
+      if e.message.start_with?(REPLY_MOVED)
         node = assign_redirection_node(e.message)
         retry_count -= 1
         retry
       elsif e.message.start_with?(REPLY_ASK)
-        raise if retry_count <= 0
-
         node = assign_asking_node(e.message)
         node.call(CMD_ASKING)
         retry_count -= 1
@@ -283,8 +298,11 @@ class RedisClient
         raise
       end
     rescue ::RedisClient::ConnectionError
+      raise if retry_count <= 0
+
       update_cluster_info!
-      raise
+      retry_count -= 1
+      retry
     end
 
     def _scan(*command, **kwargs) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
@@ -326,9 +344,13 @@ class RedisClient
       find_node(node_key)
     end
 
-    def find_node_key(*command, primary_only: false)
+    def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
       key = @command.extract_first_key(command)
-      return if key.empty?
+      if key.empty?
+        return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
+
+        return @node.replica_node_keys.sample
+      end
 
       slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
       return unless @node.slot_exists?(slot)
@@ -340,23 +362,28 @@ class RedisClient
       end
     end
 
-    def find_node(node_key)
+    def find_node(node_key, retry_count: 3)
       return @node.sample if node_key.nil?
 
       @node.find_by(node_key)
     rescue ::RedisClient::Cluster::Node::ReloadNeeded
+      raise(::RedisClient::ConnectionError, 'unstable cluster state') if retry_count <= 0
+
       update_cluster_info!(node_key)
-      @node.find_by(node_key)
+      retry_count -= 1
+      retry
     end
 
     def update_cluster_info!(node_key = nil)
-      unless node_key.nil?
-        host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
-        @config.add_node(host, port)
-      end
+      @mutex.synchronize do
+        unless node_key.nil?
+          host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
+          @config.add_node(host, port)
+        end
 
-      @node.each(&:close)
-      @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
+        @node.each(&:close)
+        @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
+      end
     end
   end
 end

data/lib/redis_client/cluster_config.rb

@@ -15,6 +15,7 @@ class RedisClient
     VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
     VALID_NODES_KEYS = %i[ssl username password host port db].freeze
     MERGE_CONFIG_KEYS = %i[ssl username password].freeze
+    IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
 
     InvalidClientConfigError = Class.new(::RedisClient::Error)
 
@@ -22,7 +23,9 @@ class RedisClient
       @replica = true & replica
       @fixed_hostname = fixed_hostname.to_s
       @node_configs = build_node_configs(nodes.dup)
+      client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
       @client_config = merge_generic_config(client_config, @node_configs)
+      @mutex = Mutex.new
     end
 
     def inspect
@@ -51,11 +54,11 @@ class RedisClient
     end
 
     def update_node(addrs)
-      @node_configs = build_node_configs(addrs)
+      @mutex.synchronize { @node_configs = build_node_configs(addrs) }
     end
 
     def add_node(host, port)
-      @node_configs << { host: host, port: port }
+      @mutex.synchronize { @node_configs << { host: host, port: port } }
     end
 
     def dup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.6
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-17 00:00:00.000000000 Z
+date: 2022-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client