redis-cluster-client 0.7.4 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82e7044600bf789f17ab9db7287beb76ebdb8c1b24c3816523da64af6a836038
-  data.tar.gz: ca55b62fc6a4252999f4fd0080cc267bfa31609eb8c9645265072bf374e61231
+  metadata.gz: 071a90a4437e12104fba1359e1a88fd64a0aa253f53ec1fdfd5c8b1fa62c63a5
+  data.tar.gz: 773243dc28547cc114730cbef7f0f971dd7bb178d4ea9a8cb2e8c4924f99bf40
 SHA512:
-  metadata.gz: fe4e0e00d82ec9afab61a96265d1f0eb6f580ce6f9b1721df1135b2b298b21c99de46f0f04cba48a5173a299896897de80037e3256ed39f9605c6f51d3c3fd0c
-  data.tar.gz: 956d3d22f40b00c6ec8608d06d33bb85e2ba0af810ffa68a30c51469423c7542eb52f33f71695dfe02a4c2e2577b4f870dd7684c920276adb205bf5df61630bd
+  metadata.gz: d345690443bc35cf19935ae9e886e0833bee49b772f21abdfb4752629b7c436100e6a367b730055fbd155d59924cfd4d76ad3df2ee1a0ed3aff65f097a82a208
+  data.tar.gz: d76e747b3eec56184f253f40f82599f492338fe86d4d3c3d30e9cda22df087e6817673b7bf9f02c07192b77322efd7d204ff60ced77be7fe2e8764cf9d104f63

data/lib/redis_client/cluster/command.rb

@@ -2,33 +2,33 @@
 
 require 'redis_client'
 require 'redis_client/cluster/errors'
+require 'redis_client/cluster/key_slot_converter'
 require 'redis_client/cluster/normalized_cmd_name'
 
 class RedisClient
   class Cluster
     class Command
-      SLOW_COMMAND_TIMEOUT = Float(ENV.fetch('REDIS_CLIENT_SLOW_COMMAND_TIMEOUT', -1))
-
       EMPTY_STRING = ''
-      LEFT_BRACKET = '{'
-      RIGHT_BRACKET = '}'
       EMPTY_HASH = {}.freeze
+      EMPTY_ARRAY = [].freeze
 
       Detail = Struct.new(
         'RedisCommand',
         :first_key_position,
+        :last_key_position,
+        :key_step,
         :write?,
         :readonly?,
         keyword_init: true
       )
 
       class << self
-        def load(nodes)
+        def load(nodes, slow_command_timeout: -1)
           cmd = errors = nil
 
           nodes&.each do |node|
             regular_timeout = node.read_timeout
-            node.read_timeout = SLOW_COMMAND_TIMEOUT > 0.0 ? SLOW_COMMAND_TIMEOUT : regular_timeout
+            node.read_timeout = slow_command_timeout > 0.0 ? slow_command_timeout : regular_timeout
             reply = node.call('COMMAND')
             node.read_timeout = regular_timeout
             commands = parse_command_reply(reply)
@@ -52,6 +52,8 @@ class RedisClient
 
             acc[row[0].downcase] = ::RedisClient::Cluster::Command::Detail.new(
               first_key_position: row[3],
+              last_key_position: row[4],
+              key_step: row[5],
               write?: row[2].include?('write'),
               readonly?: row[2].include?('readonly')
             )
@@ -67,9 +69,18 @@ class RedisClient
         i = determine_first_key_position(command)
         return EMPTY_STRING if i == 0
 
-        key = (command[i].is_a?(Array) ? command[i].flatten.first : command[i]).to_s
-        hash_tag = extract_hash_tag(key)
-        hash_tag.empty? ? key : hash_tag
+        (command[i].is_a?(Array) ? command[i].flatten.first : command[i]).to_s
+      end
+
+      def extract_all_keys(command)
+        keys_start = determine_first_key_position(command)
+        keys_end = determine_last_key_position(command, keys_start)
+        keys_step = determine_key_step(command)
+        return EMPTY_ARRAY if [keys_start, keys_end, keys_step].any?(&:zero?)
+
+        keys_end = [keys_end, command.size - 1].min
+        # use .. inclusive range because keys_end is a valid index.
+        (keys_start..keys_end).step(keys_step).map { |i| command[i] }
       end
 
       def should_send_to_primary?(command)
@@ -103,21 +114,44 @@ class RedisClient
         end
       end
 
-      def determine_optional_key_position(command, option_name) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        idx = command&.flatten&.map(&:to_s)&.map(&:downcase)&.index(option_name&.downcase)
-        idx.nil? ? 0 : idx + 1
+      # IMPORTANT: this determines the last key position INCLUSIVE of the last key -
+      # i.e. command[determine_last_key_position(command)] is a key.
+      # This is in line with what Redis returns from COMMANDS.
+      def determine_last_key_position(command, keys_start) # rubocop:disable Metrics/AbcSize
+        case name = ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_command(command)
+        when 'eval', 'evalsha', 'zinterstore', 'zunionstore'
+          # EVALSHA sha1 numkeys [key [key ...]] [arg [arg ...]]
+          # ZINTERSTORE destination numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE <SUM | MIN | MAX>]
+          command[2].to_i + 2
+        when 'object', 'memory'
+          # OBJECT [ENCODING | FREQ | IDLETIME | REFCOUNT] key
+          # MEMORY USAGE key [SAMPLES count]
+          keys_start
+        when 'migrate'
+          # MIGRATE host port <key | ""> destination-db timeout [COPY] [REPLACE] [AUTH password | AUTH2 username password] [KEYS key [key ...]]
+          command[3].empty? ? (command.length - 1) : 3
+        when 'xread', 'xreadgroup'
+          # XREAD [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]
+          keys_start + ((command.length - keys_start) / 2) - 1
+        else
+          # If there is a fixed, non-variable number of keys, don't iterate past that.
+          if @commands[name].last_key_position >= 0
+            @commands[name].last_key_position
+          else
+            command.length + @commands[name].last_key_position
+          end
+        end
       end
 
-      # @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
-      def extract_hash_tag(key)
-        key = key.to_s
-        s = key.index(LEFT_BRACKET)
-        return EMPTY_STRING if s.nil?
-
-        e = key.index(RIGHT_BRACKET, s + 1)
-        return EMPTY_STRING if e.nil?
+      def determine_key_step(command)
+        name = ::RedisClient::Cluster::NormalizedCmdName.instance.get_by_command(command)
+        # Some commands like EVALSHA have zero as the step in COMMANDS somehow.
+        @commands[name].key_step == 0 ? 1 : @commands[name].key_step
+      end
 
-        key[s + 1..e - 1]
+      def determine_optional_key_position(command, option_name) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        idx = command&.flatten&.map(&:to_s)&.map(&:downcase)&.index(option_name&.downcase)
+        idx.nil? ? 0 : idx + 1
       end
     end
   end

data/lib/redis_client/cluster/error_identification.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class RedisClient
+  class Cluster
+    module ErrorIdentification
+      def self.client_owns_error?(err, client)
+        err.is_a?(TaggedError) && err.from?(client)
+      end
+
+      module TaggedError
+        attr_accessor :config_instance
+
+        def from?(client)
+          client.config.equal?(config_instance)
+        end
+      end
+
+      module Middleware
+        def connect(config)
+          super
+        rescue RedisClient::Error => e
+          identify_error(e, config)
+          raise
+        end
+
+        def call(_command, config)
+          super
+        rescue RedisClient::Error => e
+          identify_error(e, config)
+          raise
+        end
+
+        def call_pipelined(_command, config)
+          super
+        rescue RedisClient::Error => e
+          identify_error(e, config)
+          raise
+        end
+
+        private
+
+        def identify_error(err, config)
+          err.singleton_class.include(TaggedError)
+          err.config_instance = config
+        end
+      end
+    end
+  end
+end

data/lib/redis_client/cluster/key_slot_converter.rb

@@ -3,6 +3,9 @@
 class RedisClient
   class Cluster
     module KeySlotConverter
+      EMPTY_STRING = ''
+      LEFT_BRACKET = '{'
+      RIGHT_BRACKET = '}'
       XMODEM_CRC16_LOOKUP = [
         0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
         0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
@@ -45,6 +48,9 @@ class RedisClient
       def convert(key)
         return nil if key.nil?
 
+        hash_tag = extract_hash_tag(key)
+        key = hash_tag unless hash_tag.empty?
+
         crc = 0
         key.each_byte do |b|
           crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]
@@ -52,6 +58,18 @@ class RedisClient
 
         crc % HASH_SLOTS
       end
+
+      # @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
+      def extract_hash_tag(key)
+        key = key.to_s
+        s = key.index(LEFT_BRACKET)
+        return EMPTY_STRING if s.nil?
+
+        e = key.index(RIGHT_BRACKET, s + 1)
+        return EMPTY_STRING if e.nil?
+
+        key[s + 1..e - 1]
+      end
     end
   end
 end

data/lib/redis_client/cluster/node/base_topology.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class RedisClient
+  class Cluster
+    class Node
+      class BaseTopology
+        IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
+        attr_reader :clients, :primary_clients, :replica_clients
+
+        def initialize(pool, concurrent_worker, **kwargs)
+          @pool = pool
+          @clients = {}
+          @client_options = kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
+          @concurrent_worker = concurrent_worker
+          @replications = EMPTY_HASH
+          @primary_node_keys = EMPTY_ARRAY
+          @replica_node_keys = EMPTY_ARRAY
+          @primary_clients = EMPTY_ARRAY
+          @replica_clients = EMPTY_ARRAY
+        end
+
+        def any_primary_node_key(seed: nil)
+          random = seed.nil? ? Random : Random.new(seed)
+          @primary_node_keys.sample(random: random)
+        end
+
+        def process_topology_update!(replications, options) # rubocop:disable Metrics/AbcSize
+          @replications = replications.freeze
+          @primary_node_keys = @replications.keys.sort.select { |k| options.key?(k) }.freeze
+          @replica_node_keys = @replications.values.flatten.sort.select { |k| options.key?(k) }.freeze
+
+          # Disconnect from nodes that we no longer want, and connect to nodes we're not connected to yet
+          disconnect_from_unwanted_nodes(options)
+          connect_to_new_nodes(options)
+
+          @primary_clients, @replica_clients = @clients.partition { |k, _| @primary_node_keys.include?(k) }.map(&:to_h)
+          @primary_clients.freeze
+          @replica_clients.freeze
+        end
+
+        private
+
+        def disconnect_from_unwanted_nodes(options)
+          (@clients.keys - options.keys).each do |node_key|
+            @clients.delete(node_key).close
+          end
+        end
+
+        def connect_to_new_nodes(options)
+          (options.keys - @clients.keys).each do |node_key|
+            option = options[node_key].merge(@client_options)
+            config = ::RedisClient::Cluster::Node::Config.new(scale_read: !@primary_node_keys.include?(node_key), **option)
+            client = @pool.nil? ? config.new_client : config.new_pool(**@pool)
+            @clients[node_key] = client
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis_client/cluster/node/latency_replica.rb

@@ -1,29 +1,14 @@
 # frozen_string_literal: true
 
-require 'redis_client/cluster/node/replica_mixin'
+require 'redis_client/cluster/node/base_topology'
 
 class RedisClient
   class Cluster
     class Node
-      class LatencyReplica
-        include ::RedisClient::Cluster::Node::ReplicaMixin
-
-        attr_reader :replica_clients
-
+      class LatencyReplica < BaseTopology
         DUMMY_LATENCY_MSEC = 100 * 1000 * 1000
         MEASURE_ATTEMPT_COUNT = 10
 
-        def initialize(replications, options, pool, concurrent_worker, **kwargs)
-          super
-
-          all_replica_clients = @clients.select { |k, _| @replica_node_keys.include?(k) }
-          latencies = measure_latencies(all_replica_clients, concurrent_worker)
-          @replications.each_value { |keys| keys.sort_by! { |k| latencies.fetch(k) } }
-          @replica_clients = select_replica_clients(@replications, @clients)
-          @clients_for_scanning = select_clients_for_scanning(@replications, @clients)
-          @existed_replicas = @replications.values.reject(&:empty?)
-        end
-
         def clients_for_scanning(seed: nil) # rubocop:disable Lint/UnusedMethodArgument
           @clients_for_scanning
         end
@@ -37,6 +22,17 @@ class RedisClient
           @existed_replicas.sample(random: random)&.first || any_primary_node_key(seed: seed)
         end
 
+        def process_topology_update!(replications, options)
+          super
+
+          all_replica_clients = @clients.select { |k, _| @replica_node_keys.include?(k) }
+          latencies = measure_latencies(all_replica_clients, @concurrent_worker)
+          @replications.each_value { |keys| keys.sort_by! { |k| latencies.fetch(k) } }
+          @replica_clients = select_replica_clients(@replications, @clients)
+          @clients_for_scanning = select_clients_for_scanning(@replications, @clients)
+          @existed_replicas = @replications.values.reject(&:empty?)
+        end
+
         private
 
         def measure_latencies(clients, concurrent_worker) # rubocop:disable Metrics/AbcSize

data/lib/redis_client/cluster/node/primary_only.rb

@@ -1,16 +1,11 @@
 # frozen_string_literal: true
 
+require 'redis_client/cluster/node/base_topology'
+
 class RedisClient
   class Cluster
     class Node
-      class PrimaryOnly
-        attr_reader :clients
-
-        def initialize(replications, options, pool, _concurrent_worker, **kwargs)
-          @primary_node_keys = replications.keys.sort
-          @clients = build_clients(@primary_node_keys, options, pool, **kwargs)
-        end
-
+      class PrimaryOnly < BaseTopology
         alias primary_clients clients
         alias replica_clients clients
 
@@ -29,17 +24,10 @@ class RedisClient
 
         alias any_replica_node_key any_primary_node_key
 
-        private
-
-        def build_clients(primary_node_keys, options, pool, **kwargs)
-          options.filter_map do |node_key, option|
-            next if !primary_node_keys.empty? && !primary_node_keys.include?(node_key)
-
-            option = option.merge(kwargs.reject { |k, _| ::RedisClient::Cluster::Node::IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
-            config = ::RedisClient::Cluster::Node::Config.new(**option)
-            client = pool.nil? ? config.new_client : config.new_pool(**pool)
-            [node_key, client]
-          end.to_h
+        def process_topology_update!(replications, options)
+          # Remove non-primary nodes from options (provided that we actually have any primaries at all)
+          options = options.select { |node_key, _| replications.key?(node_key) } if replications.keys.any?
+          super(replications, options)
         end
       end
     end

data/lib/redis_client/cluster/node/random_replica.rb

@@ -1,13 +1,11 @@
 # frozen_string_literal: true
 
-require 'redis_client/cluster/node/replica_mixin'
+require 'redis_client/cluster/node/base_topology'
 
 class RedisClient
   class Cluster
     class Node
-      class RandomReplica
-        include ::RedisClient::Cluster::Node::ReplicaMixin
-
+      class RandomReplica < BaseTopology
         def replica_clients
           keys = @replications.values.filter_map(&:sample)
           @clients.select { |k, _| keys.include?(k) }

data/lib/redis_client/cluster/node/random_replica_or_primary.rb

@@ -1,13 +1,11 @@
 # frozen_string_literal: true
 
-require 'redis_client/cluster/node/replica_mixin'
+require 'redis_client/cluster/node/base_topology'
 
 class RedisClient
   class Cluster
     class Node
-      class RandomReplicaOrPrimary
-        include ::RedisClient::Cluster::Node::ReplicaMixin
-
+      class RandomReplicaOrPrimary < BaseTopology
         def replica_clients
           keys = @replications.values.filter_map(&:sample)
           @clients.select { |k, _| keys.include?(k) }