redis-cluster-client 0.16.1 → 0.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e90bf81ffcf5eb68e78018e950813bdb727274c17157088645e1d58d763252a9
-  data.tar.gz: d4185f0e3d0b7d5069a9e156c607d39d0d10bd348f602a13ab53a68fa716e9fe
+  metadata.gz: 5c16432e083671229128632c116401dc9b1bd56f6b3b1e3c8f107956901d29f2
+  data.tar.gz: b464a6f87e8c952a5e4f5b22d0609a9be09b3c6c7817eb91ed779e0b142a4290
 SHA512:
-  metadata.gz: e30aa213ff6f7be6e70bcc2bc7329b8f0d9358f98169cea311c9e2afa66bf3fc28db5e8eb84776b09564ab7095d08c7744405065e7fbd2bf8fec483b4ddfba67
-  data.tar.gz: 5c8952e4eccce33fc59697e42bb135ecaaa25241e5491e1c854f9fb1f42b9ef03ae9044c83add3d21d4a2ac755d6d1d6f11eca1c7d6c41f47f7a31b15cc03d22
+  metadata.gz: 0b94204a193f0bfe56c75da819651c8c16c72bd9620c51996f2dcfc8b2c7faefc416c342ca7631772c96d4fec7a9e7c8aede9638c28fb540855603b829a1042d
+  data.tar.gz: 6d873858848aa00124a043efce6ee8fdd629516e8c5447802b079636de1dd4782b0584c84d791ed0bc7c589d66c273e985878cdea238054e60f1a5a5a2ce72c9

data/lib/redis_client/cluster/command.rb

@@ -9,18 +9,57 @@ class RedisClient
     class Command
       EMPTY_STRING = ''
       EMPTY_HASH = {}.freeze
-      EMPTY_ARRAY = [].freeze
 
-      private_constant :EMPTY_STRING, :EMPTY_HASH, :EMPTY_ARRAY
+      private_constant :EMPTY_HASH
 
-      Detail = Struct.new(
+      Spec = Struct.new(
         'RedisCommand',
         :first_key_position,
         :key_step,
         :write?,
         :readonly?,
         keyword_init: true
-      )
+      ) do
+        def extract_first_key(command)
+          i = first_key_position.to_i
+          return command[i] if i > 0
+
+          i = determine_first_key_position(command)
+          return ::RedisClient::Cluster::Command::EMPTY_STRING if i == 0
+
+          command[i]
+        end
+
+        def should_send_to_primary?
+          write?
+        end
+
+        def should_send_to_replica?
+          readonly?
+        end
+
+        private
+
+        def determine_first_key_position(command) # rubocop:disable Metrics/AbcSize
+          cmd_name = command.first
+          if cmd_name.casecmp('xread').zero?
+            determine_optional_key_position(command, 'streams')
+          elsif cmd_name.casecmp('xreadgroup').zero?
+            determine_optional_key_position(command, 'streams')
+          elsif cmd_name.casecmp('migrate').zero?
+            command[3].empty? ? determine_optional_key_position(command, 'keys') : 3
+          elsif cmd_name.casecmp('memory').zero?
+            command[1].to_s.casecmp('usage').zero? ? 2 : 0
+          else
+            0
+          end
+        end
+
+        def determine_optional_key_position(command, option_name)
+          i = command.index { |v| v.to_s.casecmp(option_name).zero? }
+          i.nil? ? 0 : i + 1
+        end
+      end
 
       class << self
         def load(nodes, slow_command_timeout: -1) # rubocop:disable Metrics/AbcSize
@@ -30,13 +69,14 @@ class RedisClient
             regular_timeout = node.read_timeout
             node.read_timeout = slow_command_timeout > 0.0 ? slow_command_timeout : regular_timeout
             reply = node.call('command')
-            node.read_timeout = regular_timeout
             commands = parse_command_reply(reply)
             cmd = ::RedisClient::Cluster::Command.new(commands)
             break
           rescue ::RedisClient::Error => e
             errors ||= []
             errors << e
+          ensure
+            node.read_timeout = regular_timeout
           end
 
           return cmd unless cmd.nil?
@@ -64,7 +104,7 @@ class RedisClient
                        else row[2].include?('write')
                        end
 
-            acc[row.first] = ::RedisClient::Cluster::Command::Detail.new(
+            acc[row.first] = ::RedisClient::Cluster::Command::Spec.new(
               first_key_position: pos,
               key_step: row[5],
               write?: writable,
@@ -78,53 +118,13 @@ class RedisClient
         @commands = commands || EMPTY_HASH
       end
 
-      def extract_first_key(command)
-        i = determine_first_key_position(command)
-        return EMPTY_STRING if i == 0
-
-        command[i]
-      end
-
-      def should_send_to_primary?(command)
-        find_command_info(command.first)&.write?
-      end
-
-      def should_send_to_replica?(command)
-        find_command_info(command.first)&.readonly?
+      def get_spec(name)
+        @commands[name] || @commands[name.to_s.downcase(:ascii)]
       end
 
       def exists?(name)
         @commands.key?(name) || @commands.key?(name.to_s.downcase(:ascii))
       end
-
-      private
-
-      def find_command_info(name)
-        @commands[name] || @commands[name.to_s.downcase(:ascii)]
-      end
-
-      def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/PerceivedComplexity
-        i = find_command_info(command.first)&.first_key_position.to_i
-        return i if i > 0
-
-        cmd_name = command.first
-        if cmd_name.casecmp('xread').zero?
-          determine_optional_key_position(command, 'streams')
-        elsif cmd_name.casecmp('xreadgroup').zero?
-          determine_optional_key_position(command, 'streams')
-        elsif cmd_name.casecmp('migrate').zero?
-          command[3].empty? ? determine_optional_key_position(command, 'keys') : 3
-        elsif cmd_name.casecmp('memory').zero?
-          command[1].to_s.casecmp('usage').zero? ? 2 : 0
-        else
-          i
-        end
-      end
-
-      def determine_optional_key_position(command, option_name)
-        i = command.index { |v| v.to_s.casecmp(option_name).zero? }
-        i.nil? ? 0 : i + 1
-      end
     end
   end
 end

data/lib/redis_client/cluster/concurrent_worker/none.rb

@@ -3,17 +3,57 @@
 class RedisClient
   class Cluster
     module ConcurrentWorker
-      class None
-        def new_group(size:)
-          ::RedisClient::Cluster::ConcurrentWorker::Group.new(
-            worker: self,
-            queue: [],
-            size: size
+      module None
+        class Group
+          Task = Struct.new(
+            'RedisClusterClientSingleThreadTask',
+            :id, :result, keyword_init: true
           )
+
+          def initialize(size:)
+            @tasks = Array.new(size)
+            @idx = 0
+          end
+
+          def push(id, *args, **kwargs, &block)
+            raise InvalidNumberOfTasks, "max size reached: #{@idx}" if @idx == @tasks.size
+
+            result = exec(*args, **kwargs, &block)
+            @tasks[@idx] = Task.new(id: id, result: result)
+            @idx += 1
+            nil
+          end
+
+          def each
+            raise InvalidNumberOfTasks, "expected: #{@tasks.size}, actual: #{@idx}" if @idx != @tasks.size
+
+            @tasks.each { |task| yield(task.id, task.result) }
+            nil
+          end
+
+          def close
+            @idx = 0
+            @tasks.clear
+            nil
+          end
+
+          def inspect
+            "#<#{self.class.name} size: #{@idx}, max: #{@tasks.size}>"
+          end
+
+          private
+
+          def exec(*args, **kwargs, &block)
+            block&.call(*args, **kwargs)
+          rescue StandardError => e
+            e
+          end
         end
 
-        def push(task)
-          task.exec
+        module_function
+
+        def new_group(size:)
+          Group.new(size: size)
         end
 
         def close; end

data/lib/redis_client/cluster/concurrent_worker.rb

@@ -73,7 +73,7 @@ class RedisClient
 
       def create(model: :none, size: 5)
         case model
-        when :none then ::RedisClient::Cluster::ConcurrentWorker::None.new
+        when :none then ::RedisClient::Cluster::ConcurrentWorker::None
         when :on_demand then ::RedisClient::Cluster::ConcurrentWorker::OnDemand.new(size: size)
         when :pooled then ::RedisClient::Cluster::ConcurrentWorker::Pooled.new(size: size)
         else raise ArgumentError, "unknown model: #{model}"

data/lib/redis_client/cluster/node/base_topology.rb

@@ -25,8 +25,7 @@ class RedisClient
         end
 
         def any_primary_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @primary_node_keys.sample(random: random)
+          @primary_node_keys.sample(random: make_random(seed))
         end
 
         def process_topology_update!(replications, options) # rubocop:disable Metrics/AbcSize
@@ -59,6 +58,11 @@ class RedisClient
             @clients[node_key] = client
           end
         end
+
+        def make_random(seed)
+          # OPTIMIZE: Figure out the most elegant way to pin a node during a pipeline or scan.
+          seed.nil? ? Random : Random.new(seed)
+        end
       end
     end
   end

data/lib/redis_client/cluster/node/latency_replica.rb

@@ -20,8 +20,7 @@ class RedisClient
         end
 
         def any_replica_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @existed_replicas.sample(random: random)&.first || any_primary_node_key(seed: seed)
+          @existed_replicas.sample(random: make_random(seed))&.first || any_primary_node_key(seed: seed)
         end
 
         def process_topology_update!(replications, options)

data/lib/redis_client/cluster/node/primary_only.rb

@@ -18,8 +18,7 @@ class RedisClient
         end
 
         def any_primary_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @primary_node_keys.sample(random: random)
+          @primary_node_keys.sample(random: make_random(seed))
         end
 
         alias any_replica_node_key any_primary_node_key

data/lib/redis_client/cluster/node/random_replica.rb

@@ -12,7 +12,7 @@ class RedisClient
         end
 
         def clients_for_scanning(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
+          random = make_random(seed)
           keys = @replications.map do |primary_node_key, replica_node_keys|
             replica_node_keys.empty? ? primary_node_key : replica_node_keys.sample(random: random)
           end
@@ -21,13 +21,13 @@ class RedisClient
         end
 
         def find_node_key_of_replica(primary_node_key, seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @replications.fetch(primary_node_key, EMPTY_ARRAY).sample(random: random) || primary_node_key
+          replica_node_keys = @replications.fetch(primary_node_key, EMPTY_ARRAY)
+          replica_node_key = replica_node_keys.size <= 1 ? replica_node_keys.first : replica_node_keys.sample(random: make_random(seed))
+          replica_node_key || primary_node_key
         end
 
         def any_replica_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @replica_node_keys.sample(random: random) || any_primary_node_key(seed: seed)
+          @replica_node_keys.sample(random: make_random(seed)) || any_primary_node_key(seed: seed)
         end
       end
     end

data/lib/redis_client/cluster/node/random_replica_or_primary.rb

@@ -12,7 +12,7 @@ class RedisClient
         end
 
         def clients_for_scanning(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
+          random = make_random(seed)
           keys = @replications.map do |primary_node_key, replica_node_keys|
             decide_use_primary?(random, replica_node_keys.size) ? primary_node_key : replica_node_keys.sample(random: random)
           end
@@ -21,7 +21,7 @@ class RedisClient
         end
 
         def find_node_key_of_replica(primary_node_key, seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
+          random = make_random(seed)
 
           replica_node_keys = @replications.fetch(primary_node_key, EMPTY_ARRAY)
           if decide_use_primary?(random, replica_node_keys.size)
@@ -32,8 +32,7 @@ class RedisClient
         end
 
         def any_replica_node_key(seed: nil)
-          random = seed.nil? ? Random : Random.new(seed)
-          @replica_node_keys.sample(random: random) || any_primary_node_key(seed: seed)
+          @replica_node_keys.sample(random: make_random(seed)) || any_primary_node_key(seed: seed)
         end
 
         private

data/lib/redis_client/cluster/node.rb

@@ -479,9 +479,11 @@ class RedisClient
       def with_reload_jitter
         return unless @next_reload_time.nil? || obtain_current_time >= @next_reload_time
 
-        yield
-
-        @next_reload_time = obtain_current_time + @random.rand(JITTER_WINDOW)
+        begin
+          yield
+        ensure
+          @next_reload_time = obtain_current_time + @random.rand(JITTER_WINDOW)
+        end
       end
 
       def with_reload_lock

data/lib/redis_client/cluster/node_key.rb

@@ -18,12 +18,30 @@ class RedisClient
       end
 
       def split(node_key)
-        pos = node_key&.rindex(DELIMITER, -1)
+        return [node_key, nil] if node_key.nil? || node_key.empty?
+
+        bracketed = split_bracketed(node_key)
+        return bracketed unless bracketed.nil?
+
+        pos = node_key.rindex(DELIMITER, -1)
         return [node_key, nil] if pos.nil?
 
         [node_key[0, pos], node_key[(pos + 1)..]]
       end
 
+      def split_bracketed(node_key)
+        return nil unless node_key.start_with?('[')
+
+        end_bracket = node_key.index(']')
+        return nil if end_bracket.nil?
+
+        host = node_key[1, end_bracket - 1]
+        remainder = node_key[(end_bracket + 1)..]
+        port = remainder.start_with?(DELIMITER) ? remainder[1..] : nil
+        [host, port]
+      end
+      private_class_method :split_bracketed
+
       def build_from_uri(uri)
         return '' if uri.nil?
 

data/lib/redis_client/cluster/optimistic_locking.rb

@@ -54,7 +54,7 @@ class RedisClient
       rescue ::RedisClient::ConnectionError
         # Deduct the number of retries that happened _inside_ router#handle_redirection from our remaining
         # _external_ retries. Always deduct at least one in case handle_redirection raises without trying the block.
-        retry_count -= [times_block_executed, 1].min
+        retry_count -= times_block_executed == 0 ? 1 : [times_block_executed, 1].min
         raise if retry_count < 0
 
         retry

data/lib/redis_client/cluster/pipeline.rb

@@ -230,10 +230,10 @@ class RedisClient
 
       def append_pipeline(node_key)
         @pipelines ||= {}
-        @pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(::RedisClient::Cluster::NoopCommandBuilder)
-        @pipelines[node_key].add_outer_index(@size)
+        pi = (@pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(::RedisClient::Cluster::NoopCommandBuilder))
+        pi.add_outer_index(@size)
         @size += 1
-        @pipelines[node_key]
+        pi
       end
 
       def do_pipelining(client, pipeline)

data/lib/redis_client/cluster/pub_sub.rb

@@ -61,7 +61,17 @@ class RedisClient
       end
 
       BUF_SIZE = Integer(ENV.fetch('REDIS_CLIENT_PUBSUB_BUF_SIZE', 1024))
-      private_constant :BUF_SIZE
+      RECOVERY_BASE_INTERVAL = Float(ENV.fetch('REDIS_CLIENT_PUBSUB_RECOVERY_INTERVAL_SEC', 1.0))
+      RECOVERY_MAX_INTERVAL = Float(ENV.fetch('REDIS_CLIENT_PUBSUB_RECOVERY_MAX_INTERVAL_SEC', 30.0))
+      RECOVERY_MAX_ATTEMPTS = Integer(ENV.fetch('REDIS_CLIENT_PUBSUB_RECOVERY_MAX_ATTEMPTS', 10))
+      SUBSCRIBE_COMMANDS = %w[subscribe psubscribe ssubscribe].freeze
+      UNSUBSCRIBE_TO_SUBSCRIBE = {
+        'unsubscribe' => 'subscribe',
+        'punsubscribe' => 'psubscribe',
+        'sunsubscribe' => 'ssubscribe'
+      }.freeze
+      private_constant :BUF_SIZE, :RECOVERY_BASE_INTERVAL, :RECOVERY_MAX_INTERVAL, :RECOVERY_MAX_ATTEMPTS,
+                       :SUBSCRIBE_COMMANDS, :UNSUBSCRIBE_TO_SUBSCRIBE
 
       def initialize(router, command_builder)
         @router = router
@@ -74,14 +84,14 @@ class RedisClient
       def call(*args, **kwargs)
         command = @command_builder.generate(args, kwargs)
         _call(command)
-        @commands << command
+        remember(command)
         nil
       end
 
       def call_v(command)
         command = @command_builder.generate(command)
         _call(command)
-        @commands << command
+        remember(command)
         nil
       end
 
@@ -118,6 +128,35 @@ class RedisClient
 
       private
 
+      def remember(command)
+        cmd = command.first.to_s.downcase
+        if SUBSCRIBE_COMMANDS.include?(cmd)
+          @commands << command
+        elsif (subscribe_cmd = UNSUBSCRIBE_TO_SUBSCRIBE[cmd])
+          forget_subscriptions(subscribe_cmd, command[1..])
+        else
+          @commands << command
+        end
+      end
+
+      def forget_subscriptions(subscribe_cmd, channels)
+        if channels.nil? || channels.empty?
+          @commands.reject! { |c| c.first.to_s.casecmp(subscribe_cmd).zero? }
+        else
+          @commands.reject! { |c| prune_entry(c, subscribe_cmd, channels) }
+        end
+      end
+
+      def prune_entry(entry, subscribe_cmd, channels)
+        return false unless entry.first.to_s.casecmp(subscribe_cmd).zero?
+
+        remaining = entry[1..] - channels
+        return true if remaining.empty?
+
+        entry.replace([entry.first, *remaining])
+        false
+      end
+
       def _call(command) # rubocop:disable Metrics/AbcSize
         if command.first.casecmp('subscribe').zero?
           call_to_single_state(command)
@@ -179,6 +218,7 @@ class RedisClient
       end
 
       def start_over
+        attempt = 0
         loop do
           @router.renew_cluster_state
           @state_dict.each_value(&:close)
@@ -186,9 +226,16 @@ class RedisClient
           @commands.each { |command| _call(command) }
           break
         rescue ::RedisClient::ConnectionError, ::RedisClient::Cluster::NodeMightBeDown
-          sleep 1.0
+          attempt += 1
+          raise if attempt >= RECOVERY_MAX_ATTEMPTS
+
+          sleep recovery_interval(attempt)
         end
       end
+
+      def recovery_interval(attempt)
+        [RECOVERY_BASE_INTERVAL * (2**(attempt - 1)), RECOVERY_MAX_INTERVAL].min
+      end
     end
   end
 end

data/lib/redis_client/cluster/router.rb

@@ -246,23 +246,27 @@ class RedisClient
       end
 
       def find_node_key(command, seed: nil)
-        key = @command.extract_first_key(command)
-        find_node_key_by_key(key, seed: seed, primary: @command.should_send_to_primary?(command))
+        cmd_spec = @command.get_spec(command.first)
+        find_node_key_by_key(
+          cmd_spec&.extract_first_key(command),
+          seed: seed,
+          primary: cmd_spec&.should_send_to_primary?
+        )
       end
 
       def find_primary_node_key(command)
-        key = @command.extract_first_key(command)
-        return nil unless key&.size&.> 0
+        key = @command.get_spec(command.first)&.extract_first_key(command)
+        return unless key&.size&.> 0
 
         find_node_key_by_key(key, primary: true)
       end
 
       def find_slot(command)
-        find_slot_by_key(@command.extract_first_key(command))
+        find_slot_by_key(@command.get_spec(command.first)&.extract_first_key(command))
       end
 
       def find_slot_by_key(key)
-        return if key.empty?
+        return if key.nil? || key.empty?
 
         ::RedisClient::Cluster::KeySlotConverter.convert(key)
       end
@@ -467,8 +471,13 @@ class RedisClient
 
         return assign_node_and_send_command(method, command, args, &block) if command.size <= keys_step + 1 || ::RedisClient::Cluster::KeySlotConverter.hash_tag_included?(command[1])
 
-        seed = @config.use_replica? && @config.replica_affinity == :random ? nil : Random.new_seed
-        pipeline = ::RedisClient::Cluster::Pipeline.new(self, @command_builder, @concurrent_worker, exception: true, seed: seed)
+        pipeline = ::RedisClient::Cluster::Pipeline.new(
+          self,
+          @command_builder,
+          @concurrent_worker,
+          exception: true,
+          seed: Random.new_seed
+        )
 
         single_command = Array.new(keys_step + 1)
         single_command[0] = single_key_cmd

data/lib/redis_client/cluster.rb

@@ -95,13 +95,12 @@ class RedisClient
     end
 
     def pipelined(exception: true)
-      seed = @config.use_replica? && @config.replica_affinity == :random ? nil : Random.new_seed
       pipeline = ::RedisClient::Cluster::Pipeline.new(
         router,
         @command_builder,
         @concurrent_worker,
         exception: exception,
-        seed: seed
+        seed: Random.new_seed
       )
 
       yield pipeline

data/lib/redis_client/cluster_config.rb

@@ -32,6 +32,12 @@ class RedisClient
 
     InvalidClientConfigError = Class.new(::RedisClient::Cluster::Error)
 
+    SENSITIVE_INSPECT_KEYS = %i[username password].freeze
+    INSPECT_REDACTED_KEYS = %i[command_builder].freeze
+    INSPECT_PLACEHOLDER = '[FILTERED]'
+
+    private_constant :SENSITIVE_INSPECT_KEYS, :INSPECT_REDACTED_KEYS, :INSPECT_PLACEHOLDER
+
     attr_reader :command_builder, :client_config, :replica_affinity, :slow_command_timeout,
                 :connect_with_original_config, :startup_nodes, :max_startup_sample, :id
 
@@ -65,7 +71,7 @@ class RedisClient
     end
 
     def inspect
-      "#<#{self.class.name} #{startup_nodes.values.map { |v| v.reject { |k| k == :command_builder } }}>"
+      "#<#{self.class.name} #{startup_nodes.values.map { |v| redact_for_inspect(v) }}>"
     end
 
     def connect_timeout
@@ -117,6 +123,14 @@ class RedisClient
 
     private
 
+    def redact_for_inspect(node_config)
+      node_config.each_with_object({}) do |(key, value), redacted|
+        next if INSPECT_REDACTED_KEYS.include?(key)
+
+        redacted[key] = SENSITIVE_INSPECT_KEYS.include?(key) ? INSPECT_PLACEHOLDER : value
+      end
+    end
+
     def merge_concurrency_option(option)
       opts = {}
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.16.3
 platform: ruby
 authors:
 - Taishi Kasuga