redis-cluster-client 0.13.5 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0abe95ee2079026038121cd93e8ae7929726ee4206b3d1f401dc7ed82f1da2c8
-  data.tar.gz: 7fa2e9c6c20d814f54e2fcf8045094869b4b5fc6c39144a8900328a9f5401abc
+  metadata.gz: e90bf81ffcf5eb68e78018e950813bdb727274c17157088645e1d58d763252a9
+  data.tar.gz: d4185f0e3d0b7d5069a9e156c607d39d0d10bd348f602a13ab53a68fa716e9fe
 SHA512:
-  metadata.gz: 26be35f5eb57e7280b8a38441f3d80c4956638a68418cf6964bdbf27d4cb3051f5955179b90066409c908ce771f3ee4286d0e569711d198ffcbfc22a70cc49bf
-  data.tar.gz: 976db1b41a2d3592dac70bf25ea8f44308824bbc7009082dd54fe68ad01b934e1b572b89df1f3b9e45b791851966c909a022defae45f4d9c3270469e9d6acc88
+  metadata.gz: e30aa213ff6f7be6e70bcc2bc7329b8f0d9358f98169cea311c9e2afa66bf3fc28db5e8eb84776b09564ab7095d08c7744405065e7fbd2bf8fec483b4ddfba67
+  data.tar.gz: 5c8952e4eccce33fc59697e42bb135ecaaa25241e5491e1c854f9fb1f42b9ef03ae9044c83add3d21d4a2ac755d6d1d6f11eca1c7d6c41f47f7a31b15cc03d22

data/lib/redis_client/cluster/concurrent_worker/pooled.rb

@@ -10,6 +10,10 @@ class RedisClient
       # It is a fixed size but we can modify the size with some environment variables.
       # So it consumes memory 1 MB multiplied a number of workers.
       class Pooled
+        IO_ERROR_NEVER = { IOError => :never }.freeze
+        IO_ERROR_IMMEDIATE = { IOError => :immediate }.freeze
+        private_constant :IO_ERROR_NEVER, :IO_ERROR_IMMEDIATE
+
         def initialize(size:)
           raise ArgumentError, "size must be positive: #{size}" unless size.positive?
 
@@ -33,7 +37,9 @@ class RedisClient
 
         def close
           @q.clear
-          @workers.each { |t| t&.exit }
+          workers = @workers.compact
+          workers.each(&:exit)
+          workers.each(&:join)
           @workers.clear
           @q.close
           @pid = nil
@@ -65,7 +71,15 @@ class RedisClient
 
         def spawn_worker
           Thread.new(@q) do |q|
-            loop { q.pop.exec }
+            Thread.handle_interrupt(IO_ERROR_NEVER) do
+              loop do
+                Thread.handle_interrupt(IO_ERROR_IMMEDIATE) do
+                  q.pop.exec
+                end
+              end
+            end
+          rescue IOError
+            # stream closed in another thread
           end
         end
       end

data/lib/redis_client/cluster/key_slot_converter.rb

@@ -70,7 +70,7 @@ class RedisClient
         e = key.index(RIGHT_BRACKET, s + 1)
         return EMPTY_STRING if e.nil?
 
-        key[s + 1..e - 1]
+        s + 1 < e ? key[s + 1, e - s - 1] : EMPTY_STRING
       end
 
       def hash_tag_included?(key)

data/lib/redis_client/cluster/node.rb

@@ -24,10 +24,12 @@ class RedisClient
       ROLE_FLAGS = %w[master slave].freeze
       EMPTY_ARRAY = [].freeze
       EMPTY_HASH = {}.freeze
-      STATE_REFRESH_INTERVAL = (3..10).freeze
+      EMPTY_STRING = ''
+      JITTER_WINDOW = (3_000_000...10_000_000).freeze # micro seconds
 
       private_constant :USE_CHAR_ARRAY_SLOT, :SLOT_SIZE, :MIN_SLOT, :MAX_SLOT,
-                       :DEAD_FLAGS, :ROLE_FLAGS, :EMPTY_ARRAY, :EMPTY_HASH
+                       :DEAD_FLAGS, :ROLE_FLAGS, :EMPTY_ARRAY, :EMPTY_HASH, :EMPTY_STRING,
+                       :JITTER_WINDOW
 
       ReloadNeeded = Class.new(::RedisClient::Cluster::Error)
 
@@ -46,7 +48,7 @@ class RedisClient
         end
 
         def serialize(str)
-          str << id << node_key << role << primary_id << config_epoch
+          str << id << node_key << role << primary_id
         end
       end
 
@@ -106,8 +108,7 @@ class RedisClient
         @topology = klass.new(pool, @concurrent_worker, **kwargs)
         @config = config
         @mutex = Mutex.new
-        @last_reloaded_at = nil
-        @reload_times = 0
+        @next_reload_time = nil
         @random = Random.new
       end
 
@@ -193,26 +194,22 @@ class RedisClient
       end
 
       def update_slot(slot, node_key)
-        return if @mutex.locked?
+        return unless @mutex.try_lock
 
-        @mutex.synchronize do
-          @slots[slot] = node_key
-        rescue RangeError
-          @slots = Array.new(SLOT_SIZE) { |i| @slots[i] }
-          @slots[slot] = node_key
-        end
+        @slots[slot] = node_key
+      rescue RangeError
+        @slots = Array.new(SLOT_SIZE) { |i| @slots[i] }
+        @slots[slot] = node_key
+      ensure
+        @mutex.unlock if @mutex.owned?
       end
 
-      def reload!
+      def try_reload!
         with_reload_lock do
-          with_startup_clients(@config.max_startup_sample) do |startup_clients|
-            @node_info = refetch_node_info_list(startup_clients)
-            @node_configs = @node_info.to_h do |node_info|
-              [node_info.node_key, @config.client_config_for_node(node_info.node_key)]
+          with_reload_jitter do
+            with_startup_clients(@config.max_startup_sample) do |clients|
+              reload!(clients)
             end
-            @slots = build_slot_node_mappings(@node_info)
-            @replications = build_replication_mappings(@node_info)
-            @topology.process_topology_update!(@replications, @node_configs)
           end
         end
       end
@@ -312,12 +309,11 @@ class RedisClient
           work_group.push(i, raw_client) do |client|
             regular_timeout = client.read_timeout
             client.read_timeout = @config.slow_command_timeout > 0.0 ? @config.slow_command_timeout : regular_timeout
-            reply = client.call_once('cluster', 'nodes')
-            client.read_timeout = regular_timeout
-            parse_cluster_node_reply(reply)
+            fetch_cluster_state(client)
           rescue StandardError => e
             e
           ensure
+            client.read_timeout = regular_timeout
             client&.close
           end
         end
@@ -347,6 +343,16 @@ class RedisClient
         grouped.max_by { |_, v| v.size }[1].first
       end
 
+      def fetch_cluster_state(client)
+        reply = client.call_once('cluster', 'shards')
+        parse_cluster_shards_reply(reply)
+      rescue ::RedisClient::CommandError => e
+        raise unless e.message.start_with?('ERR Unknown subcommand')
+
+        reply = client.call_once('cluster', 'nodes')
+        parse_cluster_node_reply(reply)
+      end
+
       def parse_cluster_node_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         reply.each_line("\n", chomp: true).filter_map do |line|
           fields = line.split
@@ -390,16 +396,19 @@ class RedisClient
               id: id,
               node_key: NodeKey.build_from_host_port(ip, arr[1]),
               role: role,
-              primary_id: role == 'master' ? nil : primary_id,
+              primary_id: role == 'master' ? EMPTY_STRING : primary_id,
               slots: role == 'master' ? slots : EMPTY_ARRAY
             )
           end
-        end.freeze
+        end
       end
 
       def parse_cluster_shards_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         reply.each_with_object([]) do |shard, acc|
+          resp2 = shard.is_a?(Array)
+          shard = shard.each_slice(2).to_h if resp2
           nodes = shard.fetch('nodes')
+          nodes = nodes.map { |n| n.each_slice(2).to_h } if resp2
           primary_id = nodes.find { |n| n.fetch('role') == 'master' }.fetch('id')
 
           nodes.each do |node|
@@ -411,18 +420,18 @@ class RedisClient
               id: node.fetch('id'),
               node_key: NodeKey.build_from_host_port(ip, node['port'] || node['tls-port']),
               role: role == 'master' ? role : 'slave',
-              primary_id: role == 'master' ? nil : primary_id,
+              primary_id: role == 'master' ? EMPTY_STRING : primary_id,
               slots: role == 'master' ? shard.fetch('slots').each_slice(2).to_a.freeze : EMPTY_ARRAY
             )
           end
-        end.freeze
+        end
       end
 
       # As redirection node_key is dependent on `cluster-preferred-endpoint-type` config,
       # node_key should use hostname if present in CLUSTER NODES output.
       #
       # See https://redis.io/commands/cluster-nodes/ for details on the output format.
-      # node_address matches fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]>
+      # node_address matches the format: <ip:port@cport[,hostname[,auxiliary_field=value]*]>
       def parse_node_key(node_address)
         ip_chunk, hostname, _auxiliaries = node_address.split(',')
         ip_port_string = ip_chunk.split('@').first
@@ -432,6 +441,16 @@ class RedisClient
         "#{hostname}:#{port}"
       end
 
+      def reload!(clients)
+        @node_info = refetch_node_info_list(clients)
+        @node_configs = @node_info.to_h do |node_info|
+          [node_info.node_key, @config.client_config_for_node(node_info.node_key)]
+        end
+        @slots = build_slot_node_mappings(@node_info)
+        @replications = build_replication_mappings(@node_info)
+        @topology.process_topology_update!(@replications, @node_configs)
+      end
+
       def with_startup_clients(count) # rubocop:disable Metrics/AbcSize
         if @config.connect_with_original_config
           # If connect_with_original_config is set, that means we need to build actual client objects
@@ -457,35 +476,41 @@ class RedisClient
         end
       end
 
+      def with_reload_jitter
+        return unless @next_reload_time.nil? || obtain_current_time >= @next_reload_time
+
+        yield
+
+        @next_reload_time = obtain_current_time + @random.rand(JITTER_WINDOW)
+      end
+
       def with_reload_lock
-        # What should happen with concurrent calls #reload? This is a realistic possibility if the cluster goes into
+        # What should happen with concurrent calls #try_reload! This is a realistic possibility if the cluster goes into
         # a CLUSTERDOWN state, and we're using a pooled backend. Every thread will independently discover this, and
-        # call reload!.
-        # For now, if a reload is in progress, wait for that to complete, and consider that the same as us having
-        # performed the reload.
-        # Probably in the future we should add a circuit breaker to #reload itself, and stop trying if the cluster is
+        # call #try_reload!.
+        # For now, if a reload is in progress by a thread, the other threads do not wait for that to complete, and
+        # they throw an error.
+        # Probably in the future we should add a circuit breaker to #try_reload! itself, and stop trying if the cluster is
         # obviously not working.
-        wait_start = obtain_current_time
-        @mutex.synchronize do
-          return if @last_reloaded_at && @last_reloaded_at > wait_start
-
-          if @last_reloaded_at && @reload_times > 1
-            # Mitigate load of servers by naive logic. Don't sleep with exponential backoff.
-            now = obtain_current_time
-            elapsed = @last_reloaded_at + @random.rand(STATE_REFRESH_INTERVAL) * 1_000_000
-            return if now < elapsed
-          end
+        return unless @mutex.try_lock
 
-          r = yield
-          @last_reloaded_at = obtain_current_time
-          @reload_times += 1
-          r
-        end
+        yield
+      ensure
+        @mutex.unlock if @mutex.owned?
       end
 
       def obtain_current_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
       end
+
+      def bypass_reload!
+        # DO NOT USE THIS METHOD
+        with_reload_lock do
+          with_startup_clients(@config.max_startup_sample) do |clients|
+            reload!(clients)
+          end
+        end
+      end
     end
   end
 end

data/lib/redis_client/cluster/pub_sub.rb

@@ -7,6 +7,10 @@ class RedisClient
   class Cluster
     class PubSub
       class State
+        IO_ERROR_NEVER = { IOError => :never }.freeze
+        IO_ERROR_IMMEDIATE = { IOError => :immediate }.freeze
+        private_constant :IO_ERROR_NEVER, :IO_ERROR_IMMEDIATE
+
         def initialize(client, queue)
           @client = client
           @worker = nil
@@ -22,7 +26,11 @@ class RedisClient
         end
 
         def close
-          @worker.exit if @worker&.alive?
+          if @worker&.alive?
+            @worker.exit
+            @worker.join
+          end
+
           @client.close
         rescue ::RedisClient::ConnectionError
           # ignore
@@ -35,21 +43,24 @@ class RedisClient
           # It is a fixed size but we can modify the size with some environment variables.
           # So it consumes memory 1 MB multiplied a number of workers.
           Thread.new(client, queue, nil) do |pubsub, q, prev_err|
-            loop do
-              q << pubsub.next_event
-              prev_err = nil
-            rescue StandardError => e
-              next sleep 0.005 if e.instance_of?(prev_err.class) && e.message == prev_err&.message
-
-              q << e
-              prev_err = e
+            Thread.handle_interrupt(IO_ERROR_NEVER) do
+              loop do
+                Thread.handle_interrupt(IO_ERROR_IMMEDIATE) { q << pubsub.next_event }
+                prev_err = nil
+              rescue StandardError => e
+                next sleep 0.005 if e.instance_of?(prev_err.class) && e.message == prev_err&.message
+
+                Thread.handle_interrupt(IO_ERROR_IMMEDIATE) { q << e }
+                prev_err = e
+              end
             end
+          rescue IOError
+            # stream closed in another thread
           end
         end
       end
 
       BUF_SIZE = Integer(ENV.fetch('REDIS_CLIENT_PUBSUB_BUF_SIZE', 1024))
-
       private_constant :BUF_SIZE
 
       def initialize(router, command_builder)

data/lib/redis_client/cluster/router.rb

@@ -17,7 +17,6 @@ class RedisClient
     class Router
       ZERO_CURSOR_FOR_SCAN = '0'
       TSF = ->(f, x) { f.nil? ? x : f.call(x) }.curry
-      Ractor.make_shareable(TSF) if Object.const_defined?(:Ractor, false) && Ractor.respond_to?(:make_shareable)
       DEDICATED_ACTIONS = lambda do # rubocop:disable Metrics/BlockLength
         action = Struct.new('RedisCommandRoutingAction', :method_name, :reply_transformer, keyword_init: true)
         pick_first = ->(reply) { reply.first } # rubocop:disable Style/SymbolProc
@@ -59,6 +58,7 @@ class RedisClient
           'bgsave' => all_node_first_action,
           'quit' => all_node_first_action,
           'save' => all_node_first_action,
+          'select' => all_node_first_action,
           'flushall' => primary_first_action,
           'flushdb' => primary_first_action,
           'readonly' => not_supported_action,
@@ -84,7 +84,7 @@ class RedisClient
         @pool = pool
         @client_kwargs = kwargs
         @node = ::RedisClient::Cluster::Node.new(concurrent_worker, config: config, pool: pool, **kwargs)
-        @node.reload!
+        @node.try_reload!
         @command = ::RedisClient::Cluster::Command.load(@node.replica_clients.shuffle, slow_command_timeout: config.slow_command_timeout)
         @command_builder = @config.command_builder
       rescue ::RedisClient::Cluster::InitialSetupError => e
@@ -93,9 +93,8 @@ class RedisClient
       end
 
       def send_command(method, command, *args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        return assign_node_and_send_command(method, command, args, &block) unless DEDICATED_ACTIONS.key?(command.first)
-
         action = DEDICATED_ACTIONS[command.first]
+        return assign_node_and_send_command(method, command, args, &block) if action.nil?
         return send(action.method_name, method, command, args, &block) if action.reply_transformer.nil?
 
         reply = send(action.method_name, method, command, args)
@@ -167,8 +166,8 @@ class RedisClient
       rescue ::RedisClient::ConnectionError => e
         raise unless ::RedisClient::Cluster::ErrorIdentification.client_owns_error?(e, node)
 
-        retry_count -= 1
         renew_cluster_state
+        retry_count -= 1
 
         if retry_count >= 0
           # Find the node to use for this command - if this fails for some reason, though, re-use
@@ -180,7 +179,6 @@ class RedisClient
           retry
         end
 
-        retry if retry_count >= 0
         raise
       end
 
@@ -294,7 +292,7 @@ class RedisClient
       end
 
       def renew_cluster_state
-        @node.reload!
+        @node.try_reload!
       rescue ::RedisClient::Cluster::InitialSetupError
         # ignore
       end
@@ -340,8 +338,8 @@ class RedisClient
         raise if retry_count <= 0
         raise if e.errors.values.none? { |err| err.message.include?('WAIT cannot be used with replica instances') }
 
-        retry_count -= 1
         renew_cluster_state
+        retry_count -= 1
         retry
       end
 
@@ -382,35 +380,23 @@ class RedisClient
       end
 
       def send_cluster_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if command[1].casecmp('addslots').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('delslots').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('failover').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('forget').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('meet').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('replicate').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('reset').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('set-config-epoch').zero?
-          fail_not_supported_command(method, command, args, &block)
-        elsif command[1].casecmp('setslot').zero?
-          fail_not_supported_command(method, command, args, &block)
+        if command[1].casecmp('keyslot').zero? ||
+           command[1].casecmp('info').zero? ||
+           command[1].casecmp('nodes').zero? ||
+           command[1].casecmp('slots').zero? ||
+           command[1].casecmp('shards').zero? ||
+           command[1].casecmp('count-failure-reports').zero? ||
+           command[1].casecmp('slaves').zero?
+          assign_node(command).public_send(method, *args, command, &block)
         elsif command[1].casecmp('saveconfig').zero?
           @node.call_all(method, command, args).first.then(&TSF.call(block))
-        elsif command[1].casecmp('getkeysinslot').zero?
-          raise ArgumentError, command.join(' ') if command.size != 4
-
+        elsif command[1].casecmp('countkeysinslot').zero? || command[1].casecmp('getkeysinslot').zero?
           handle_node_reload_error do
-            node_key = @node.find_node_key_of_replica(command[2])
+            node_key = @node.find_node_key_of_replica(command[2]) || @node.any_replica_node_key
             @node.find_by(node_key).public_send(method, *args, command, &block)
           end
         else
-          assign_node(command).public_send(method, *args, command, &block)
+          fail_not_supported_command(method, command, args, &block)
         end
       end
 
@@ -464,7 +450,7 @@ class RedisClient
       end
 
       def send_multiple_keys_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        # This implementation is prioritized performance rather than readability or so.
+        # This implementation prioritizes performance over readability.
         cmd = command.first
         if cmd.casecmp('mget').zero?
           single_key_cmd = 'get'
@@ -514,8 +500,8 @@ class RedisClient
       rescue ::RedisClient::Cluster::Node::ReloadNeeded
         raise ::RedisClient::Cluster::NodeMightBeDown.new.with_config(@config) if retry_count <= 0
 
-        retry_count -= 1
         renew_cluster_state
+        retry_count -= 1
         retry
       end
     end

data/lib/redis_client/cluster.rb

@@ -130,8 +130,26 @@ class RedisClient
       ::RedisClient::Cluster::PubSub.new(router, @command_builder)
     end
 
-    def with(...)
-      raise NotImplementedError, 'No way to use'
+    # Compatibility layer for RedisClient::Pooled
+    def with(_options = nil)
+      yield self
+    end
+    alias then with
+
+    # Compatibility layer for RedisClient::HashRing
+    def node_for(_key)
+      self
+    end
+
+    # Compatibility layer for RedisClient::HashRing
+    def nodes_for(*keys)
+      keys.flatten!
+      { self => keys }
+    end
+
+    # Compatibility layer for RedisClient::HashRing
+    def nodes
+      [self].freeze
     end
 
     def close

data/lib/redis_client/cluster_config.rb

@@ -15,16 +15,15 @@ class RedisClient
     DEFAULT_SCHEME = 'redis'
     SECURE_SCHEME = 'rediss'
     DEFAULT_NODE = "#{DEFAULT_SCHEME}://#{DEFAULT_HOST}:#{DEFAULT_PORT}"
-    Ractor.make_shareable(DEFAULT_NODE) if Object.const_defined?(:Ractor, false) && Ractor.respond_to?(:make_shareable)
     DEFAULT_NODES = [DEFAULT_NODE].freeze
     VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
     VALID_NODES_KEYS = %i[ssl username password host port db].freeze
-    MERGE_CONFIG_KEYS = %i[ssl username password].freeze
+    MERGE_CONFIG_KEYS = %i[ssl username password db].freeze
     IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
     MAX_WORKERS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', -1)) # for backward compatibility
-    # It's used with slow queries of fetching meta data like CLUSTER NODES, COMMAND and so on.
+    # Used for slow commands that fetch metadata, e.g. CLUSTER NODES, COMMAND.
     SLOW_COMMAND_TIMEOUT = Float(ENV.fetch('REDIS_CLIENT_SLOW_COMMAND_TIMEOUT', -1))
-    # It affects to strike a balance between load and stability in initialization or changed states.
+    # Controls the balance between startup load and stability during initialization or cluster state changes.
     MAX_STARTUP_SAMPLE = Integer(ENV.fetch('REDIS_CLIENT_MAX_STARTUP_SAMPLE', 3))
 
     private_constant :DEFAULT_HOST, :DEFAULT_PORT, :DEFAULT_SCHEME, :SECURE_SCHEME, :DEFAULT_NODES,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster-client
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.16.1
 platform: ruby
 authors:
 - Taishi Kasuga
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.24'
+        version: '0.28'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.24'
+        version: '0.28'
 email:
 - proxy0721@gmail.com
 executables: []
@@ -74,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Redis cluster-aware client for Ruby
 test_files: []