redis-cluster 0.0.9 → 1.0.0.pre.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2fc9fa13ac023fafd595e13fe442e70e7d83b95
-  data.tar.gz: d577599a4bc2952ab2817fd202896caa8407bf23
+  metadata.gz: 1dd6f66183d627b2454e9e7ba629274ba32a1e78
+  data.tar.gz: 3519af542e7ed85abb0593bccbb9dba0f451762d
 SHA512:
-  metadata.gz: cdbaa05a0d4bf1cd7b3f5bdb770cfc3349572aa96b1be80d7672e374917404ed75c7d815c4b667a0d35cce369597d5e3bd45528a161cf58c0bf827d72ad5fc51
-  data.tar.gz: a451ee0b177399a81aabbca94c6204acf336c6d82369ed3a476b3620c1192aaf4d6f948d2a238d6701cd5dea1921b48be6aaf2a6ae790d44de57cfe9a0a93805
+  metadata.gz: f8c89e900077de31226af2067769576136735356388dd69d4f528697a3f67d4e2fa6685def24ba27a24a5f57992cf91eca2002a61c49e9baa4c94ae7c582c100
+  data.tar.gz: 74921e593e6a1338e2bcc382f6e27718b3d8a4808dd7ed8d589d539b3d3f9fce72ccac5bd0f0d002f1b142b2b04a0fe2ef3966f77fa92a68165ed1a2c01b0d6d

data/README.md

@@ -33,6 +33,10 @@ SRE Bukalapak
                              redis_opts: { timeout: 5, connect_timeout: 1 },
                              cluster_opts: { force_cluster: false, read_mode: :master_slave, silent: true, logger: Logger.new }
                            )
+   redis.middlewares.register(:commit) do |*args, &block|
+     puts "this is RedisCluster middlewares"
+     block.call
+   end
    ````
 
 ### Development Guide
@@ -41,7 +45,7 @@ SRE Bukalapak
    See [here](https://rvm.io/rvm/install) to install `rvm`.
    And run these commands to install `bundler` and other dependencies
 
-   ````
+   ````sh
    gem install bundler
    bundle install
    ````
@@ -53,20 +57,20 @@ SRE Bukalapak
 
 4. Make your change and it's test.
 
-   ````
+   ````sh
    vim lib/**.rb
    vim spec/**_spec.rb
    ````
 
 5. Optionally, run the test in your local
 
-   ````
+   ````sh
    rake # run all test and lint
    ````
 
 6. Commit and push your change to upstream
 
-   ````
+   ````sh
    git commit -m "message"
    git push # add "--set-upstream branch_name" after "git push" if you haven't set the upstream
    ````
@@ -91,6 +95,68 @@ Option for RedisCluster.
 - `silent`: whether or not RedisCluster will raise error.
 - `logger`: if specified. RedisCluster will log all of RedisCluster errors here.
 
+#### Middlewares
+
+Middlewares are hooks that RedisCluster provide to observe RedisCluster events. To register a middlewares, provide callable object (object that respond to call)
+or give block in register method. Middlewares must give block result as return value.
+````ruby
+class Callable
+  call
+    start = Time.now
+    yield
+  rescue StandardError => e
+    raise e
+  ensure
+    Metrics.publish(elapsed_time: Time.now - start)
+  end
+end
+redis.middlewares.register(:commit, Callable.new)
+
+redis.middlewares.register(:commit) do |*args, &block|
+  begin
+    res = block.call
+  rescue StandardError => e
+    Log.warn('failed')
+    raise e
+  end
+  Log.info('success')
+  res
+end
+````
+
+Currently there are 3 events that RedisCluster publish.
+- `:commit`
+  RedisCluster will fire `:commit` events when RedisCluster::Client call redis server. It give queue of command as arguments.
+  ````ruby
+  redis.middlewares.register(:commit) do |queues, &block|
+    puts 'this is :commit events'
+    puts "first command: #{queues.first.first}
+    puts "last command: #{queues.last.first}
+    block.call
+  end
+  ````
+- `:call`
+  This events is fired when command is issued in RedisCluster client before any load balancing is done. It give call arguments as arguments
+  ````ruby
+  redis.middlewares.register(:call) do |keys, command, opts = {}, &block|
+    puts "keys to load balance: #{keys}"
+    puts "redis command: #{command.first}"
+    block.call
+  end
+  redis.get('something')
+  # Output:
+  #   keys to load balance: something
+  #   redis command: get
+  ````
+- `:pipelined`
+  This events is fired when pipelined method is called from redis client. It does not give any arguments
+  ````ruby
+  redis.middlewares.register(:pipelined) do |&block|
+    puts 'pipelined is called'
+    block.call
+  end
+  ````
+
 ### Limitation
 
 All multi keys operation, cluster command, multi-exec, and some commands are not supported.

data/lib/redis-cluster.rb

@@ -1,40 +1,37 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 require_relative 'redis_cluster/cluster'
 require_relative 'redis_cluster/client'
 require_relative 'redis_cluster/future'
 require_relative 'redis_cluster/function'
+require_relative 'redis_cluster/middlewares'
 
 # RedisCluster is a client for redis-cluster *huh*
 class RedisCluster
   include MonitorMixin
   include Function
 
-  attr_reader :cluster, :options
+  attr_reader :cluster, :cluster_opts, :redis_opts, :middlewares
 
   def initialize(seeds, redis_opts: nil, cluster_opts: nil)
-    @options = cluster_opts || {}
-    cluster_options = redis_opts || {}
-    @cluster = Cluster.new(seeds, cluster_options.merge(force_cluster: force_cluster?))
+    @cluster_opts = cluster_opts || {}
+    @redis_opts = redis_opts || {}
+    @middlewares = Middlewares.new
+
+    client_creater = method(:create_client)
+    @cluster = Cluster.new(seeds, cluster_opts, &client_creater)
 
     super()
   end
 
   def logger
-    options[:logger]
+    cluster_opts[:logger]
   end
 
   def silent?
-    options[:silent]
-  end
-
-  def read_mode
-    options[:read_mode] || :master
-  end
-
-  def force_cluster?
-    options[:force_cluster]
+    cluster_opts[:silent]
   end
 
   def connected?
@@ -49,9 +46,25 @@ class RedisCluster
     !@pipeline.nil?
   end
 
-  def call(keys, command, opts = {})
+  def call(*args, &block)
+    middlewares.invoke(:call, *args) do
+      _call(*args, &block)
+    end
+  end
+
+  def pipelined(*args, &block)
+    middlewares.invoke(:pipelined, *args) do
+      _pipelined(*args, &block)
+    end
+  end
+
+  private
+
+  NOOP = ->(v){ v }
+
+  def _call(keys, command, opts = {})
     opts[:transform] ||= NOOP
-    slot = slot_for([ keys ].flatten )
+    slot = cluster.slot_for(keys)
 
     safely do
       if pipeline?
@@ -62,7 +75,7 @@ class RedisCluster
     end
   end
 
-  def pipelined
+  def _pipelined
     safely do
       return yield if pipeline?
 
@@ -94,11 +107,6 @@ class RedisCluster
     end
   end
 
-  private
-
-  NOOP = ->(v){ v }
-  CROSSSLOT_ERROR = Redis::CommandError.new("CROSSSLOT Keys in request don't hash to the same slot")
-
   def safely
     synchronize{ yield } if block_given?
   rescue StandardError => e
@@ -106,17 +114,12 @@ class RedisCluster
     raise e unless silent?
   end
 
-  def slot_for(keys)
-    slot = keys.map{ |k| cluster.slot_for(k) }.uniq
-    slot.size == 1 ? slot.first : ( raise CROSSSLOT_ERROR )
-  end
-
   def call_immediately(slot, command, transform:, read: false)
     try = 3
     asking = false
     reply = nil
-    mode = read ? read_mode : :master
-    client = cluster.public_send(mode, slot)
+    mode = read ? :read : :write
+    client = cluster.client_for(mode, slot)
 
     while try.positive?
       begin
@@ -134,7 +137,7 @@ class RedisCluster
       rescue Redis::CannotConnectError => e
         asking = false
         cluster.reset
-        client = cluster.public_send(mode, slot)
+        client = cluster.client_for(mode, slot)
         reply = e
       end
     end
@@ -151,11 +154,11 @@ class RedisCluster
   def map_pipeline(pipe)
     futures = ::Hash.new{ |h, k| h[k] = [] }
     pipe.each do |future|
-      url = future.url || cluster.master(future.slot).url
+      url = future.url || cluster.client_for(:write, future.slot).url
       futures[url] << future
     end
 
-    return futures
+    futures
   end
 
   def do_pipelined(url, futures)
@@ -192,12 +195,12 @@ class RedisCluster
       leftover << future
     end
 
-    return [leftover, moved]
+    [leftover, moved]
   rescue Redis::CannotConnectError
     # reset url and asking when connection refused
     futures.each{ |f| f.url = nil; f.asking = false }
 
-    return [futures, true]
+    [futures, true]
   end
 
   def scan_reply(reply)
@@ -210,4 +213,11 @@ class RedisCluster
       raise reply
     end
   end
+
+  def create_client(url)
+    host, port = url.split(':', 2)
+    Client.new(redis_opts.merge(host: host, port: port)).tap do |c|
+      c.middlewares = middlewares
+    end
+  end
 end

data/lib/redis_cluster/client.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 require_relative 'version'
@@ -9,6 +10,7 @@ class RedisCluster
   # useful addition
   class Client
     attr_reader :client, :queue, :url
+    attr_accessor :middlewares
 
     def initialize(opts)
       @client = Redis::Client.new(opts)
@@ -38,6 +40,16 @@ class RedisCluster
     end
 
     def commit
+      return _commit unless middlewares
+
+      middlewares.invoke(:commit, queue.dup) do
+        _commit
+      end
+    end
+
+    private
+
+    def _commit
       return nil if queue.empty?
 
       result = Array.new(queue.size)
@@ -48,7 +60,7 @@ class RedisCluster
       end
       @queue = []
 
-      return result
+      result
     end
   end
 end

data/lib/redis_cluster/cluster.rb

@@ -1,49 +1,50 @@
 # frozen_string_literal: true
+
 require_relative 'client'
 
 class RedisCluster
 
   # Cluster implement redis cluster logic for client.
   class Cluster
-    attr_reader :options, :slots, :clients, :replicas
+    attr_reader :options, :slots, :clients, :replicas, :client_creater
 
     HASH_SLOTS = 16_384
+    CROSSSLOT_ERROR = Redis::CommandError.new("CROSSSLOT Keys in request don't hash to the same slot")
 
-    def initialize(seeds, options = {})
-      @options = options
+    def initialize(seeds, cluster_opts = {}, &block)
+      @options = cluster_opts
       @slots = []
       @clients = {}
       @replicas = nil
+      @client_creater = block
 
       init_client(seeds)
     end
 
     def force_cluster?
-      options[:force_cluster]
+      options[:force_cluster] || false
     end
 
-    # Return Redis::Client for a given key.
-    # Modified from https://github.com/antirez/redis-rb-cluster/blob/master/cluster.rb#L104-L117
-    def slot_for(key)
-      key = key.to_s
-      if (s = key.index('{'))
-        if (e = key.index('}', s + 1)) && e != s+1
-          key = key[s+1..e-1]
-        end
-      end
-      crc16(key) % HASH_SLOTS
+    def read_mode
+      options[:read_mode] || :master
     end
 
-    def master(slot)
-      slots[slot].first
+    def slot_for(keys)
+      slot = [keys].flatten.map{ |k| _slot_for(k) }.uniq
+      slot.size == 1 ? slot.first : (raise CROSSSLOT_ERROR)
     end
 
-    def slave(slot)
-      slots[slot][1..-1].sample || slots[slot].first
-    end
+    def client_for(operation, slot)
+      mode = operation == :read ? read_mode : :master
 
-    def master_slave(slot)
-      slots[slot].sample
+      case mode
+      when :master
+        slots[slot].first
+      when :slave
+        slots[slot][1..-1].sample || slots[slot].first
+      when :master_slave
+        slots[slot].sample
+      end
     end
 
     def close
@@ -66,7 +67,7 @@ class RedisCluster
         slots_and_clients(client)
       rescue StandardError => e
         clients.delete(client.url)
-        try.positive? ? retry : ( raise e )
+        try.positive? ? retry : (raise e)
       end
     end
 
@@ -74,12 +75,16 @@ class RedisCluster
       clients[url] ||= create_client(url)
     end
 
+    def inspect
+      "#<RedisCluster cluster v#{RedisCluster::VERSION}>"
+    end
+
     private
 
     def slots_and_clients(client)
       replicas = ::Hash.new{ |h, k| h[k] = [] }
 
-      result = client.call([:cluster, :slots])
+      result = client.call(%i[cluster slots])
       if result.is_a?(StandardError)
         if result.message.eql?('ERR This instance has cluster support disabled') &&
            !force_cluster?
@@ -124,11 +129,12 @@ class RedisCluster
     end
 
     def create_client(url)
-      host, port = url.split(':', 2)
-      client_options = options.clone.tap do |opts|
-        opts.delete(:force_update)
+      if client_creater
+        client_creater.call(url)
+      else
+        host, port = url.split(':', 2)
+        Client.new(host: host, port: port)
       end
-      Client.new(client_options.merge(host: host, port: port))
     end
 
     # -----------------------------------------------------------------------------
@@ -152,11 +158,23 @@ class RedisCluster
     def crc16(bytes)
       crc = 0
       bytes.each_byte do |b|
-        crc = ((crc<<8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc>>8)^b) & 0xff]
+        crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]
       end
       crc
     end
 
+    # Return Redis::Client for a given key.
+    # Modified from https://github.com/antirez/redis-rb-cluster/blob/master/cluster.rb#L104-L117
+    def _slot_for(key)
+      key = key.to_s
+      if (s = key.index('{'))
+        if (e = key.index('}', s + 1)) && e != s + 1
+          key = key[s + 1..e - 1]
+        end
+      end
+      crc16(key) % HASH_SLOTS
+    end
+
     XMODEM_CRC16_LOOKUP = [
       0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
       0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
@@ -190,6 +208,6 @@ class RedisCluster
       0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
       0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
       0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
-    ]
+    ].freeze
   end
 end

data/lib/redis_cluster/function/hash.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/function/key.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/function/scan.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster
@@ -73,7 +74,7 @@ class RedisCluster
       # @param [Hash] options
       #   - `:match => String`: only return keys matching the pattern
       #   - `:count => Integer`: return count keys at most per iteration
-      [:zscan, :hscan, :sscan].each do |method|
+      %i[zscan hscan sscan].each do |method|
         define_method "#{method}_each" do |key, options = {}, &block|
           return if block.nil?
 

data/lib/redis_cluster/function/set.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/function/sorted_set.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/function/string.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/future.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'redis'
 
 class RedisCluster

data/lib/redis_cluster/middlewares.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class RedisCluster
+
+  # Middlewares is collection for RedisCluster middleware.
+  class Middlewares
+    attr_reader :middlewares
+
+    def initialize
+      @middlewares = Hash.new{ |h, k| h[k] = [] }
+    end
+
+    def register(name, callable = nil, &block)
+      return if !callable && !block_given?
+
+      middlewares[name] << (callable || block)
+    end
+
+    def invoke(name, *args, &block)
+      callback = middlewares[name].reduce(block) do |acc, obs|
+        proc{ obs.call(*args, &acc) }
+      end
+
+      callback.call
+    end
+  end
+end

data/lib/redis_cluster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class RedisCluster
-  VERSION = '0.0.9'
+  VERSION = '1.0.0-rc.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-cluster
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 1.0.0.pre.rc.1
 platform: ruby
 authors:
 - Bukalapak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-02 00:00:00.000000000 Z
+date: 2018-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -44,6 +44,7 @@ files:
 - lib/redis_cluster/function/sorted_set.rb
 - lib/redis_cluster/function/string.rb
 - lib/redis_cluster/future.rb
+- lib/redis_cluster/middlewares.rb
 - lib/redis_cluster/version.rb
 homepage: https://github.com/bukalapak/redis-cluster
 licenses:
@@ -60,9 +61,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1