redis-cluster-client 0.0.2 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 374dde2480377552fd2178d4940d0f9f9b4473c4308c89fa210e2b0ac8f36d6d
-  data.tar.gz: 5b22559b68cd287904a17f853a4cfd6a0fc0bf53418eeca3988987a585a55b71
+  metadata.gz: 6fce5f4a15cfcab3d7cca37052e9f85ad45abcee8f4de9e458fc560a01382842
+  data.tar.gz: 2da9fb7a3ae8836dd7606fbfd05e33f423529716493d3bb8fccd6d01bae10bc3
 SHA512:
-  metadata.gz: da95dc7179899376c736d6e5b75a04b9f8d91d0b97095b2478c9b96879f0f4ff01d75ffe86bf3efdebb2af0e13b6397a3afd41a1e6857fc41097f53ac8cdacf2
-  data.tar.gz: 6759c62ea25c92e8160540b5edb354a03253d7d43199ccbf0479d1b19070b90cb17877b376012186ff063316f28f16157d8fbd1e0bb3b67c3168ed92d6afa31a
+  metadata.gz: fd4b929261fe0a50f4b13fe4963550e7b8c1197ee4f813a464b309140b29d7cc75a95259f100481d2f7e5a958a98608aca980e8308dca125200c1acb4e506e06
+  data.tar.gz: b41237f574ca12b3826e8d869f3ea3251a0acb41961fb009efe7bb5d2f07517b17cae80c5161da91603b4586a63beef11ba618b6386cfd684012cc7fa09752d1

data/lib/redis_client/cluster/node.rb

@@ -12,6 +12,8 @@ class RedisClient
       SLOT_SIZE = 16_384
       MIN_SLOT = 0
       MAX_SLOT = SLOT_SIZE - 1
+      IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
+
       ReloadNeeded = Class.new(::RedisClient::Error)
 
       class Config < ::RedisClient::Config
@@ -77,6 +79,7 @@ class RedisClient
         @slots = build_slot_node_mappings(node_info)
         @replications = build_replication_mappings(node_info)
         @clients = build_clients(options, pool: pool, **kwargs)
+        @mutex = Mutex.new
       end
 
       def inspect
@@ -99,6 +102,12 @@ class RedisClient
         @clients.filter_map { |k, _| primary?(k) ? k : nil }.sort
       end
 
+      def replica_node_keys
+        return primary_node_keys if replica_disabled?
+
+        @clients.filter_map { |k, _| replica?(k) ? k : nil }.sort
+      end
+
       def find_by(node_key)
         @clients.fetch(node_key)
       rescue KeyError
@@ -120,24 +129,17 @@ class RedisClient
       def call_replica(method, *command, **kwargs, &block)
         return call_primary(method, *command, **kwargs, &block) if replica_disabled?
 
+        replica_node_keys = @replications.values.map(&:sample)
         try_map do |node_key, client|
-          next if primary?(node_key)
+          next if primary?(node_key) || !replica_node_keys.include?(node_key)
 
           client.send(method, *command, **kwargs, &block)
         end.values
       end
 
-      # TODO: impl
-      def process_all(commands, &block)
-        try_map { |_, client| client.process(commands, &block) }.values
-      end
-
       def scale_reading_clients
-        clients = @clients.select do |node_key, _|
-          replica_disabled? ? primary?(node_key) : replica?(node_key)
-        end
-
-        clients.values.sort_by do |client|
+        keys = replica_disabled? ? @replications.keys : @replications.values.map(&:first)
+        @clients.select { |k, _| keys.include?(k) }.values.sort_by do |client|
           ::RedisClient::Cluster::NodeKey.build_from_host_port(client.config.host, client.config.port)
         end
       end
@@ -166,7 +168,7 @@ class RedisClient
       end
 
       def update_slot(slot, node_key)
-        @slots[slot] = node_key
+        @mutex.synchronize { @slots[slot] = node_key }
       end
 
       private
@@ -180,7 +182,7 @@ class RedisClient
       end
 
       def replica?(node_key)
-        !(@replications.nil? || @replications.size.zero?) && @replications[node_key].size.zero?
+        !(@replications.nil? || @replications.size.zero?) && !@replications.key?(node_key)
       end
 
       def build_slot_node_mappings(node_info)
@@ -199,7 +201,6 @@ class RedisClient
         node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
           primary_info = dict[info[:primary_id]]
           acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
-          acc[info[:node_key]]
         end
       end
 
@@ -207,7 +208,10 @@ class RedisClient
         options.filter_map do |node_key, option|
           next if replica_disabled? && replica?(node_key)
 
-          config = ::RedisClient::Cluster::Node::Config.new(scale_read: replica?(node_key), **option.merge(kwargs))
+          config = ::RedisClient::Cluster::Node::Config.new(
+            scale_read: replica?(node_key),
+            **option.merge(kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
+          )
           client = pool.nil? ? config.new_client : config.new_pool(**pool)
 
           [node_key, client]
@@ -217,15 +221,17 @@ class RedisClient
       def try_map # rubocop:disable Metrics/MethodLength
         errors = {}
         results = {}
-
-        @clients.each do |node_key, client|
-          reply = yield(node_key, client)
-          results[node_key] = reply unless reply.nil?
-        rescue ::RedisClient::CommandError => e
-          errors[node_key] = e
-          next
+        threads = @clients.map do |k, v|
+          Thread.new(k, v) do |node_key, client|
+            Thread.pass
+            reply = yield(node_key, client)
+            results[node_key] = reply unless reply.nil?
+          rescue ::RedisClient::CommandError => e
+            errors[node_key] = e
+          end
         end
 
+        threads.each(&:join)
         return results if errors.empty?
 
         raise ::RedisClient::Cluster::CommandErrorCollection, errors

data/lib/redis_client/cluster/node_key.rb

@@ -19,7 +19,7 @@ class RedisClient
         pos = node_key&.rindex(DELIMITER, -1)
         return [node_key, nil] if pos.nil?
 
-        [node_key[0, pos], node_key[pos + 1, node_key.size - pos - 1]]
+        [node_key[0, pos], node_key[(pos + 1)..]]
       end
 
       def build_from_uri(uri)

data/lib/redis_client/cluster.rb

@@ -15,24 +15,23 @@ class RedisClient
       def initialize(client)
         @client = client
         @grouped = Hash.new([].freeze)
-        @replies = []
         @size = 0
       end
 
       def call(*command, **kwargs)
-        node_key = @client.send(:find_node_key, command, primary_only: true)
+        node_key = @client.send(:find_node_key, *command, primary_only: true)
         @grouped[node_key] += [[@size, :call, command, kwargs]]
         @size += 1
       end
 
       def call_once(*command, **kwargs)
-        node_key = @client.send(:find_node_key, command, primary_only: true)
+        node_key = @client.send(:find_node_key, *command, primary_only: true)
         @grouped[node_key] += [[@size, :call_once, command, kwargs]]
         @size += 1
       end
 
       def blocking_call(timeout, *command, **kwargs)
-        node_key = @client.send(:find_node_key, command, primary_only: true)
+        node_key = @client.send(:find_node_key, *command, primary_only: true)
         @grouped[node_key] += [[@size, :blocking_call, timeout, command, kwargs]]
         @size += 1
       end
@@ -41,31 +40,64 @@ class RedisClient
         @size.zero?
       end
 
-      # TODO: use concurrency
       def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-        @grouped.each do |node_key, rows|
-          node_key = node_key.nil? ? @client.instance_variable_get(:@node).primary_node_keys.sample : node_key
-          replies = @client.send(:find_node, node_key).pipelined do |pipeline|
-            rows.each do |row|
-              case row[1]
-              when :call then pipeline.call(*row[2], **row[3])
-              when :call_once then pipeline.call_once(*row[2], **row[3])
-              when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
-              else raise NotImplementedError, row[1]
+        all_replies = Array.new(@size)
+        threads = @grouped.map do |k, v|
+          Thread.new(@client, k, v) do |client, node_key, rows|
+            Thread.pass
+            replies = client.send(:find_node, node_key).pipelined do |pipeline|
+              rows.each do |row|
+                case row[1]
+                when :call then pipeline.call(*row[2], **row[3])
+                when :call_once then pipeline.call_once(*row[2], **row[3])
+                when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
+                else raise NotImplementedError, row[1]
+                end
               end
             end
-          end
 
-          raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
+            raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
 
-          rows.each_with_index { |row, idx| @replies[row.first] = replies[idx] }
+            rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
+          end
         end
 
-        @replies
+        threads.each(&:join)
+        all_replies
+      end
+    end
+
+    class PubSub
+      def initialize(client)
+        @client = client
+        @pubsub = nil
+      end
+
+      def call(*command, **kwargs)
+        close
+        @pubsub = @client.send(:assign_node, *command).pubsub
+        @pubsub.call(*command, **kwargs)
+      end
+
+      def close
+        @pubsub&.close
+        @pubsub = nil
+      end
+
+      def next_event(timeout = nil)
+        @pubsub&.next_event(timeout)
       end
     end
 
     ZERO_CURSOR_FOR_SCAN = '0'
+    CMD_SCAN = 'SCAN'
+    CMD_SSCAN = 'SSCAN'
+    CMD_HSCAN = 'HSCAN'
+    CMD_ZSCAN = 'ZSCAN'
+    CMD_ASKING = 'ASKING'
+    REPLY_OK = 'OK'
+    REPLY_MOVED = 'MOVED'
+    REPLY_ASK = 'ASK'
 
     def initialize(config, pool: nil, **kwargs)
       @config = config.dup
@@ -73,6 +105,7 @@ class RedisClient
       @client_kwargs = kwargs
       @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
       @command = ::RedisClient::Cluster::Command.load(@node)
+      @mutex = Mutex.new
     end
 
     def inspect
@@ -97,35 +130,27 @@ class RedisClient
 
       cursor = ZERO_CURSOR_FOR_SCAN
       loop do
-        cursor, keys = _scan('SCAN', cursor, *args, **kwargs)
+        cursor, keys = _scan(CMD_SCAN, cursor, *args, **kwargs)
         keys.each(&block)
         break if cursor == ZERO_CURSOR_FOR_SCAN
       end
     end
 
     def sscan(key, *args, **kwargs, &block)
-      node = assign_node('SSCAN', key)
+      node = assign_node(CMD_SSCAN, key)
       try_send(node, :sscan, key, *args, **kwargs, &block)
     end
 
     def hscan(key, *args, **kwargs, &block)
-      node = assign_node('HSCAN', key)
+      node = assign_node(CMD_HSCAN, key)
       try_send(node, :hscan, key, *args, **kwargs, &block)
     end
 
     def zscan(key, *args, **kwargs, &block)
-      node = assign_node('ZSCAN', key)
+      node = assign_node(CMD_ZSCAN, key)
       try_send(node, :zscan, key, *args, **kwargs, &block)
     end
 
-    def mset
-      # TODO: impl
-    end
-
-    def mget
-      # TODO: impl
-    end
-
     def pipelined
       pipeline = ::RedisClient::Cluster::Pipeline.new(self)
       yield pipeline
@@ -135,17 +160,8 @@ class RedisClient
     end
 
     def pubsub
-      # TODO: impl
-    end
-
-    def size
-      # TODO: impl
-    end
-
-    def with(options = {})
-      # TODO: impl
+      ::RedisClient::Cluster::PubSub.new(self)
     end
-    alias then with
 
     def close
       @node.each(&:close)
@@ -253,17 +269,17 @@ class RedisClient
     def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       node.send(method, *args, **kwargs, &block)
     rescue ::RedisClient::CommandError => e
-      if e.message.start_with?('MOVED')
+      if e.message.start_with?(REPLY_MOVED)
         raise if retry_count <= 0
 
         node = assign_redirection_node(e.message)
         retry_count -= 1
         retry
-      elsif e.message.start_with?('ASK')
+      elsif e.message.start_with?(REPLY_ASK)
         raise if retry_count <= 0
 
         node = assign_asking_node(e.message)
-        node.call('ASKING')
+        node.call(CMD_ASKING)
         retry_count -= 1
         retry
       else
@@ -309,13 +325,17 @@ class RedisClient
     end
 
     def assign_node(*command)
-      node_key = find_node_key(command)
+      node_key = find_node_key(*command)
       find_node(node_key)
     end
 
-    def find_node_key(command, primary_only: false)
+    def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
       key = @command.extract_first_key(command)
-      return if key.empty?
+      if key.empty?
+        return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
+
+        return @node.replica_node_keys.sample
+      end
 
       slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
       return unless @node.slot_exists?(slot)
@@ -337,13 +357,15 @@ class RedisClient
     end
 
     def update_cluster_info!(node_key = nil)
-      unless node_key.nil?
-        host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
-        @config.add_node(host, port)
-      end
+      @mutex.synchronize do
+        unless node_key.nil?
+          host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
+          @config.add_node(host, port)
+        end
 
-      @node.each(&:close)
-      @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
+        @node.each(&:close)
+        @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
+      end
     end
   end
 end

data/lib/redis_client/cluster_config.rb

@@ -15,6 +15,7 @@ class RedisClient
     VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
     VALID_NODES_KEYS = %i[ssl username password host port db].freeze
     MERGE_CONFIG_KEYS = %i[ssl username password].freeze
+    IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
 
     InvalidClientConfigError = Class.new(::RedisClient::Error)
 
@@ -22,7 +23,9 @@ class RedisClient
       @replica = true & replica
       @fixed_hostname = fixed_hostname.to_s
       @node_configs = build_node_configs(nodes.dup)
+      client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
       @client_config = merge_generic_config(client_config, @node_configs)
+      @mutex = Mutex.new
     end
 
     def inspect
@@ -51,11 +54,11 @@ class RedisClient
     end
 
     def update_node(addrs)
-      @node_configs = build_node_configs(addrs)
+      @mutex.synchronize { @node_configs = build_node_configs(addrs) }
     end
 
     def add_node(host, port)
-      @node_configs << { host: host, port: port }
+      @mutex.synchronize { @node_configs << { host: host, port: port } }
     end
 
     def dup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.5
 platform: ruby
 authors:
 - Taishi Kasuga
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-16 00:00:00.000000000 Z
+date: 2022-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client