redis-cluster 1.0.1.pre.rc.1 → 1.1.0.pre.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f8427709e6b67eaf1266d6e330dfc37958fbb10dc0ab3845ee97a14319466ce
-  data.tar.gz: f26b971ccc78221ef3a3a1c6da1207e71d8b594d3c955ee529743abea5fb4c08
+  metadata.gz: 2520c212f0cb77a948aef0fe024f39d00cdb4231723e8a308bcf5247c5e1c6ac
+  data.tar.gz: 20ce3bd10f0d341040e99bc64e685c885689ecb9d35c72c447f76a9f91897f10
 SHA512:
-  metadata.gz: bc342af53829102b0da2108ca999ca74473d62e54adeb3238e898f1e899a4510d5785b07c838b22a282dce5c65e59a8187f34cc3fc6595957dc36289d4595778
-  data.tar.gz: 95221ec6964ab2aac0c639a4da59a0f501489e97ad5e27eb27be5f61a5c5158d96ae7c425eb9145b9360df06bd198b8510d96e8cbd2ed1226109ee98300bd6c9
+  metadata.gz: f64e6681e26e57b3a5d29244788f68b7ec45f847de472b0fecc5a5c5658621be096a5f03f6c6f5c145e249f6a25ddd9afc189dfd021aed9236ecca0e7fabfa6c
+  data.tar.gz: 576b6c1c881c10bf58b45aaaca92441e2985a5d8925b862eb854a18b4798046f2910778ae5284d3115184f532d003e73079e69be2bf302f935d2447d4e6fefe8

data/README.md

@@ -94,6 +94,9 @@ Option for RedisCluster.
 - `read_mode`: for read command, RedisClient can try to read from slave if specified. Supported option is `:master`(default), `:slave`, and `:master_slave`.
 - `silent`: whether or not RedisCluster will raise error.
 - `logger`: if specified. RedisCluster will log all of RedisCluster errors here.
+- `reset_interval`: reset threshold. A reset can only happen once per reset_interval.
+- `circuit_threshold`: Threshold how many error that client will considered an unhealthy.
+- `circuit_interval`: How long failed count will be remembered.
 
 #### Middlewares
 

data/lib/redis-cluster.rb

@@ -4,6 +4,7 @@ require 'redis'
 
 require_relative 'redis_cluster/cluster'
 require_relative 'redis_cluster/client'
+require_relative 'redis_cluster/circuit'
 require_relative 'redis_cluster/future'
 require_relative 'redis_cluster/function'
 require_relative 'redis_cluster/middlewares'
@@ -87,17 +88,20 @@ class RedisCluster
         while !@pipeline.empty? && try.positive?
           try -= 1
           moved = false
+          down = false
           mapped_future = map_pipeline(@pipeline)
 
           @pipeline = []
           mapped_future.each do |url, futures|
             leftover, error = do_pipelined(url, futures)
-            moved ||= (error == :moved || error == :down)
+            moved ||= error == :moved
+            down ||= error == :down
 
             @pipeline.concat(leftover)
           end
 
-          cluster.reset if moved
+          # force if moved, do not force if down
+          cluster.reset(force: moved) if moved || down
         end
 
         @pipeline.first.value unless @pipeline.empty?
@@ -115,36 +119,28 @@ class RedisCluster
   end
 
   def call_immediately(slot, command, transform:, read: false)
-    try = 3
-    asking = false
-    reply = nil
     mode = read ? :read : :write
     client = cluster.client_for(mode, slot)
 
-    while try.positive?
-      begin
-        try -= 1
-
-        client.push([:asking]) if asking
-        reply = client.call(command)
+    # first attempt
+    reply = client.call(command)
+    err, url = scan_reply(reply)
+    return transform.call(reply) unless err
 
-        err, url = scan_reply(reply)
-        return transform.call(reply) unless err
+    # make adjustment for cluster change
+    cluster.reset(force: true) if err == :moved
+    client = cluster[url]
 
-        cluster.reset if err == :moved
-        asking = err == :ask
-        client = cluster[url]
-      rescue NodeUnhealthyError, Redis::CannotConnectError => e
-        if e.is_a?(Redis::CannotConnectError)
-          asking = false
-          cluster.reset
-        end
-        client = cluster.client_for(mode, slot)
-        reply = e
-      end
-    end
+    # second attempt
+    client.push([:asking]) if err == :ask
+    reply = client.call(command)
+    err, = scan_reply(reply)
+    raise err if err
 
-    raise reply
+    transform.call(reply)
+  rescue LoadingStateError, CircuitOpenError, Redis::BaseConnectionError => e
+    cluster.reset
+    raise e
   end
 
   def call_pipeline(slot, command, opts)
@@ -171,6 +167,7 @@ class RedisCluster
     idx = 0
     client = cluster[url]
 
+    # map reverse index for pipeline commands.
     futures.each_with_index do |future, i|
       if future.asking
         client.push([:asking])
@@ -198,11 +195,11 @@ class RedisCluster
     end
 
     [leftover, error]
-  rescue NodeUnhealthyError, Redis::CannotConnectError => e
+  rescue LoadingStateError, CircuitOpenError, Redis::BaseConnectionError
     # reset url and asking when connection refused
     futures.each{ |f| f.url = nil; f.asking = false }
 
-    [futures, e.is_a?(NodeUnhealthyError) ? :loading : :down]
+    [futures, :down]
   end
 
   def scan_reply(reply)
@@ -220,6 +217,7 @@ class RedisCluster
     host, port = url.split(':', 2)
     Client.new(redis_opts.merge(host: host, port: port)).tap do |c|
       c.middlewares = middlewares
+      c.circuit = Circuit.new(cluster_opts[:circuit_threshold].to_f, cluster_opts[:circuit_interval].to_f)
     end
   end
 end

data/lib/redis_cluster/circuit.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class RedisCluster
+
+  # Circuit is circuit breaker for RedisCluster.
+  class Circuit
+
+    attr_reader :fail_count, :ban_until
+
+    def initialize(threshold, interval)
+      @ban_until = Time.now
+      @fail_count = 0
+      @last_fail_time = Time.now
+      @fail_threshold = threshold
+      @interval_time = interval
+    end
+
+    # Failed is a method to add failed count and compare it to threshold,
+    # Will trip circuit if the count goes through threshold.
+    #
+    # @return[void]
+    def failed
+      @fail_count = 0 if @last_fail_time + (@interval_time * 1.5) < Time.now
+      @fail_count += 1
+      @last_fail_time = Time.now
+      open! if @fail_count >= @fail_threshold
+    end
+
+    # Open! is a method to update ban time.
+    #
+    # @return[void]
+    def open!
+      @ban_until = Time.now + @interval_time
+    end
+
+    # Open? is a method to check if the circuit breaker status.
+    #
+    # @return[Boolean] Wheter the circuit is open or not
+    def open?
+      @ban_until > Time.now
+    end
+
+  end
+end

data/lib/redis_cluster/client.rb

@@ -6,25 +6,27 @@ require_relative 'version'
 
 class RedisCluster
 
-  class NodeUnhealthyError < StandardError; end
+  # LoadingStateError is an error when try to read redis that in loading state.
+  class LoadingStateError < StandardError; end
+
+  # CircuitOpenError is an error that fired when circuit in client is trip.
+  class CircuitOpenError < StandardError; end
 
   # Client is a decorator object for Redis::Client. It add queue to support pipelining and another
   # useful addition
   class Client
     attr_reader :client, :queue, :url
-    attr_accessor :middlewares
+    attr_accessor :middlewares, :circuit, :role, :refresh
 
     def initialize(opts)
       @client = Redis::Client.new(opts)
       @queue = []
       @url = "#{client.host}:#{client.port}"
-
-      @healthy = true
-      @ban_from = nil
+      @ready = false
     end
 
     def inspect
-      "#<RedisCluster client v#{RedisCluster::VERSION} for #{url}>"
+      "#<RedisCluster client v#{RedisCluster::VERSION} for #{url} (#{role} at #{refresh}) status #{healthy? ? 'healthy' : 'unhealthy'}>"
     end
 
     def connected?
@@ -52,21 +54,23 @@ class RedisCluster
       end
     end
 
+    # Healthy? will retrun true if circuit breaker is not open and Redis Client is ready.
+    # Redis Client will be ready if it already sent with `readonly` command.
+    #
+    # return [Boolean] Whether client is healthy or not.
     def healthy?
-      return true if @healthy
-
-      # ban for 60 seconds for unhealthy state
-      if Time.now - @ban_from > 60
-        @healthy = true
-        @ban_from = nil
-      end
+      return false if @circuit.open?
+      prepare unless @ready
 
-      @healthy
+      true
+    rescue StandardError
+      false
     end
 
     private
 
     def _commit
+      raise CircuitOpenError, "Circuit open in client #{url} until #{@circuit.ban_until}" if @circuit.open?
       return nil if queue.empty?
 
       result = Array.new(queue.size)
@@ -74,26 +78,26 @@ class RedisCluster
         queue.size.times do |i|
           result[i] = client.read
 
-          unhealthy!(result[i]) if error?(result[i])
+          if result[i].is_a?(Redis::CommandError) && result[i].message['LOADING']
+            @circuit.open!
+            raise LoadingStateError, "Client #{url} is in Loading State"
+          end
         end
       end
 
       result
+    rescue LoadingStateError, CircuitOpenError, Redis::BaseConnectionError => e
+      @circuit.failed
+      @ready = false if @circuit.open?
+
+      [e] # return this
     ensure
       @queue = []
     end
 
-    def unhealthy!(cause)
-      @healthy = false
-      @ban_from = Time.now
-
-      error = NodeUnhealthyError.new("Node #{@url} is unhealthy: #{cause}")
-      error.set_backtrace(cause.backtrace)
-      raise error
-    end
-
-    def error?(res)
-      res.is_a?(Redis::CommandError) && (res.message['LOADING'] || res.message['CLUSTERDOWN'])
+    def prepare
+      call([:readonly])
+      @ready = true
     end
   end
 end

data/lib/redis_cluster/cluster.rb

@@ -16,10 +16,8 @@ class RedisCluster
       @slots = []
       @clients = {}
       @replicas = nil
-      @client_creater = block || proc do |url|
-        host, port = url.split(':', 2)
-        Client.new(host: host, port: port)
-      end
+      @client_creater = block
+      @last_reset = Time.now - reset_interval
 
       @buffer = []
       init_client(seeds)
@@ -33,8 +31,11 @@ class RedisCluster
       options[:read_mode] || :master
     end
 
-    def logger
-      options[:logger]
+    # Reset_interval return interval in second which reset can happen. A reset can only happen once per reset_interval.
+    #
+    # @return [Fixnum] reset interval
+    def reset_interval
+      options[:reset_interval].to_f
     end
 
     def slot_for(keys)
@@ -67,15 +68,28 @@ class RedisCluster
       clients.values.sample
     end
 
-    def reset
+    # Reset will reload cluster topology. Reset will only be executed once per reset_interval if not forced.
+    #
+    # @param [Boolean] force: Whether to force reset to happen or not.
+    # @return [void]
+    def reset(force: false)
+      return if !force && @last_reset + reset_interval > Time.now
+
       try = 3
+      # binding.pry
+      pool = clients.values.select(&:healthy?)
       begin
         try -= 1
-        client = random
+        raise 'No healthy seed' if pool.length.zero?
+
+        i = rand(pool.length)
+        client = pool[i]
         slots_and_clients(client)
       rescue StandardError => e
+        pool.delete_at(i)
         try.positive? ? retry : (raise e)
       end
+      @last_reset = Time.now
     end
 
     def [](url)
@@ -89,18 +103,20 @@ class RedisCluster
     private
 
     def pick_client(pool, skip: 0)
-      buff_len = 0
+      unhealthy_count = 0
 
       (skip...pool.length).each do |i|
-        next unless pool[i].healthy?
-
-        @buffer[buff_len] = pool[i]
-        buff_len += 1
+        if pool[i].healthy?
+          @buffer[i - skip] = pool[i]
+        else
+          unhealthy_count += 1
+        end
       end
 
-      return nil if buff_len.zero?
+      buffer_length = pool.length - skip - unhealthy_count
+      return nil if buffer_length.zero?
 
-      i = rand(buff_len)
+      i = rand(buffer_length)
       @buffer[i]
     end
 
@@ -122,11 +138,9 @@ class RedisCluster
         arr[2..-1].each_with_index do |a, i|
           cli = self["#{a[0]}:#{a[1]}"]
           replicas[arr[0]] << cli
-          begin
-            cli.call([:readonly]) if i.nonzero?
-          rescue NodeUnhealthyError => e
-            logger&.error(e)
-          end
+
+          cli.role = i.zero? ? :master : :slave
+          cli.refresh = Time.now
         end
 
         (arr[0]..arr[1]).each do |slot|
@@ -139,21 +153,12 @@ class RedisCluster
     end
 
     def init_client(seeds)
-      try = seeds.count
-      err = nil
-
-      while try.positive?
-        try -= 1
-        begin
-          client = create_client(seeds[try])
-          slots_and_clients(client)
-          return
-        rescue StandardError => e
-          err = e
-        end
+      # register seeds into clients
+      seeds.each do |s|
+        self[s]
       end
 
-      raise err
+      reset(force: true)
     end
 
     def create_client(url)

data/lib/redis_cluster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class RedisCluster
-  VERSION = '1.0.1-rc.1'
+  VERSION = '1.1.0-rc.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster
 version: !ruby/object:Gem::Version
-  version: 1.0.1.pre.rc.1
+  version: 1.1.0.pre.rc.1
 platform: ruby
 authors:
 - Bukalapak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2019-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -33,6 +33,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redis-cluster.rb
+- lib/redis_cluster/circuit.rb
 - lib/redis_cluster/client.rb
 - lib/redis_cluster/cluster.rb
 - lib/redis_cluster/function.rb
@@ -66,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: Redis cluster client. Support pipelining.