yam-redis-with-retries 2.2.2.1

data/lib/redis/hash_ring.rb

@@ -0,0 +1,131 @@
+require 'zlib'
+
+class Redis
+  class HashRing
+
+    POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+    attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+    # nodes is a list of objects that have a proper to_s representation.
+    # replicas indicates how many virtual points should be used pr. node,
+    # replicas are required to improve the distribution.
+    def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+      @replicas = replicas
+      @ring = {}
+      @nodes = []
+      @sorted_keys = []
+      nodes.each do |node|
+        add_node(node)
+      end
+    end
+
+    # Adds a `node` to the hash ring (including a number of replicas).
+    def add_node(node)
+      @nodes << node
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node.id}:#{i}")
+        @ring[key] = node
+        @sorted_keys << key
+      end
+      @sorted_keys.sort!
+    end
+
+    def remove_node(node)
+      @nodes.reject!{|n| n.id == node.id}
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node.id}:#{i}")
+        @ring.delete(key)
+        @sorted_keys.reject! {|k| k == key}
+      end
+    end
+
+    # get the node in the hash ring for this key
+    def get_node(key)
+      get_node_pos(key)[0]
+    end
+
+    def get_node_pos(key)
+      return [nil,nil] if @ring.size == 0
+      crc = Zlib.crc32(key)
+      idx = HashRing.binary_search(@sorted_keys, crc)
+      return [@ring[@sorted_keys[idx]], idx]
+    end
+
+    def iter_nodes(key)
+      return [nil,nil] if @ring.size == 0
+      node, pos = get_node_pos(key)
+      @sorted_keys[pos..-1].each do |k|
+        yield @ring[k]
+      end
+    end
+
+    class << self
+
+      # gem install RubyInline to use this code
+      # Native extension to perform the binary search within the hashring.
+      # There's a pure ruby version below so this is purely optional
+      # for performance.  In testing 20k gets and sets, the native
+      # binary search shaved about 12% off the runtime (9sec -> 8sec).
+      begin
+        require 'inline'
+        inline do |builder|
+          builder.c <<-EOM
+          int binary_search(VALUE ary, unsigned int r) {
+              int upper = RARRAY_LEN(ary) - 1;
+              int lower = 0;
+              int idx = 0;
+
+              while (lower <= upper) {
+                  idx = (lower + upper) / 2;
+
+                  VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                  unsigned int l = NUM2UINT(continuumValue);
+                  if (l == r) {
+                      return idx;
+                  }
+                  else if (l > r) {
+                      upper = idx - 1;
+                  }
+                  else {
+                      lower = idx + 1;
+                  }
+              }
+              if (upper < 0) {
+                upper = RARRAY_LEN(ary) - 1;
+              }
+              return upper;
+          }
+          EOM
+        end
+      rescue Exception => e
+        # Find the closest index in HashRing with value <= the given value
+        def binary_search(ary, value, &block)
+          upper = ary.size - 1
+          lower = 0
+          idx = 0
+
+          while(lower <= upper) do
+            idx = (lower + upper) / 2
+            comp = ary[idx] <=> value
+
+            if comp == 0
+              return idx
+            elsif comp > 0
+              upper = idx - 1
+            else
+              lower = idx + 1
+            end
+          end
+
+          if upper < 0
+            upper = ary.size - 1
+          end
+          return upper
+        end
+
+      end
+    end
+
+  end
+end

data/lib/redis/pipeline.rb

@@ -0,0 +1,34 @@
+class Redis
+  class Pipeline
+    attr :commands
+
+    def initialize
+      @commands = []
+    end
+
+    # Starting with 2.2.1, assume that this method is called with a single
+    # array argument. Check its size for backwards compat.
+    def call(*args)
+      if args.first.is_a?(Array) && args.size == 1
+        command = args.first
+      else
+        command = args
+      end
+
+      @commands << command
+      nil
+    end
+
+    # Assume that this method is called with a single array argument. No
+    # backwards compat here, since it was introduced in 2.2.2.
+    def call_without_reply(command)
+      @commands.push command
+      nil
+    end
+
+    def call_pipelined(commands, options = {})
+      @commands.concat commands
+      nil
+    end
+  end
+end

data/lib/redis/retry.rb

@@ -0,0 +1,128 @@
+class Redis
+  class Retry
+
+    class Error < RuntimeError
+    end
+
+    class MarkedDeadError < Redis::Retry::Error
+      def initialize(delay, marked_dead_at)
+        duration_since_dead = Time.now.to_i - marked_dead_at
+        super("Connection is marked as dead. Delay: #{delay} Dead for #{duration_since_dead}")
+      end
+    end
+
+    class MaxRetriesReachedError < Redis::Retry::Error
+      attr :retries
+      def initialize(retries, error)
+        super("Reached maximum of #{retries} retries. Original Exception: #{error}")
+        @retries = retries
+      end
+    end
+
+    DEFAULT_OPTS = {
+      :with_retry        => true, # noise - used for upstream detection
+      :max_retries       => 2,    # number of times to try before giving up
+      :mark_dead         => true, # mark dead after :max_retries
+      :mark_dead_seconds => 60    # how long to mark dead, default: 60
+    }.freeze
+
+    def initialize(client, opts)
+      opts = DEFAULT_OPTS.merge(opts)
+      @client             = client
+      @max_retries        = opts[:max_retries]
+      @mark_dead          = opts[:mark_dead]
+      @mark_dead_seconds  = opts[:mark_dead_seconds]
+
+      ## we use this so we can log when requests timeout and are marked dead
+      @on_retry           = opts[:on_retry]
+      @on_max_retries     = opts[:on_max_retries]
+
+      ## initialize
+      @consecutive_errors = 0
+      @marked_dead_at     = nil
+    end
+
+    def execute(&block)
+      mark_alive_if_possible
+      retries = 0
+
+      begin
+        retval = block.call
+        @consecutive_errors = 0
+        retval
+
+      rescue Errno::ECONNREFUSED,
+             Errno::ECONNRESET,
+             Errno::EPIPE,
+             Errno::ECONNABORTED,
+             Errno::EBADF,
+             Errno::EAGAIN,
+             Timeout::Error => ex
+
+        disconnect_client
+        @consecutive_errors += 1
+
+        if @max_retries > 0
+          if should_try_again?
+            @on_retry.call(ex) if @on_retry
+            retry
+          else
+            if ! @mark_dead
+              max_retries_error = MaxRetriesReachedError.new(@consecutive_errors, ex)
+              @on_max_retries.call(max_retries_error) if @on_max_retries
+              raise max_retries_error
+            else
+              mark_dead!
+              marked_dead_error = MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at)
+              @on_max_retries.call(marked_dead_error) if @on_max_retries
+              raise marked_dead_error
+            end
+          end
+        end
+
+        ## worst case
+        raise MaxRetriesReachedError.new(1, ex)
+      end
+    end
+
+    def mark_alive_if_possible
+      if marked_dead? 
+        if marked_dead_and_ready_to_be_resuscitated? 
+          mark_alive!
+        else
+          raise MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at) if @mark_dead
+        end
+      end
+    end
+
+    ## the idea behind setting @consecutive_errors to one belowe @max_retries is
+    ## that if the redis server is still down, we only have to handle a single 
+    ## error before moving the server back to the graveyard, reducing the cost
+    ## of revival by n-1.
+    def mark_alive!
+      @consecutive_errors = @max_retries - 1
+      @marked_dead_at     = nil
+    end
+
+    def mark_dead!
+      @marked_dead_at = Time.now.to_i
+    end
+
+    def marked_dead?
+      ! @marked_dead_at.nil?
+    end
+
+    def marked_dead_and_ready_to_be_resuscitated?
+      (marked_dead?) && (Time.now.to_i - @marked_dead_at >= @mark_dead_seconds)
+    end
+
+    def should_try_again?
+      @consecutive_errors < @max_retries
+    end
+
+    def disconnect_client
+      @client.disconnect 
+    end
+
+  end
+end

data/lib/redis/subscribe.rb

@@ -0,0 +1,94 @@
+class Redis
+  class SubscribedClient
+    def initialize(client)
+      @client = client
+    end
+
+    # Starting with 2.2.1, assume that this method is called with a single
+    # array argument. Check its size for backwards compat.
+    def call(*args)
+      if args.first.is_a?(Array) && args.size == 1
+        command = args.first
+      else
+        command = args
+      end
+
+      @client.process([command])
+    end
+
+    # Assume that this method is called with a single array argument. No
+    # backwards compat here, since it was introduced in 2.2.2.
+    def call_without_reply(command)
+      @commands.push command
+      nil
+    end
+
+    def subscribe(*channels, &block)
+      subscription("subscribe", "unsubscribe", channels, block)
+    end
+
+    def psubscribe(*channels, &block)
+      subscription("psubscribe", "punsubscribe", channels, block)
+    end
+
+    def unsubscribe(*channels)
+      call [:unsubscribe, *channels]
+    end
+
+    def punsubscribe(*channels)
+      call [:punsubscribe, *channels]
+    end
+
+  protected
+
+    def subscription(start, stop, channels, block)
+      sub = Subscription.new(&block)
+
+      begin
+        @client.call_loop([start, *channels]) do |line|
+          type, *rest = line
+          sub.callbacks[type].call(*rest)
+          break if type == stop && rest.last == 0
+        end
+      ensure
+        send(stop)
+      end
+    end
+  end
+
+  class Subscription
+    attr :callbacks
+
+    def initialize
+      @callbacks = Hash.new do |hash, key|
+        hash[key] = lambda { |*_| }
+      end
+
+      yield(self)
+    end
+
+    def subscribe(&block)
+      @callbacks["subscribe"] = block
+    end
+
+    def unsubscribe(&block)
+      @callbacks["unsubscribe"] = block
+    end
+
+    def message(&block)
+      @callbacks["message"] = block
+    end
+
+    def psubscribe(&block)
+      @callbacks["psubscribe"] = block
+    end
+
+    def punsubscribe(&block)
+      @callbacks["punsubscribe"] = block
+    end
+
+    def pmessage(&block)
+      @callbacks["pmessage"] = block
+    end
+  end
+end

data/lib/redis/version.rb

@@ -0,0 +1,3 @@
+class Redis
+  VERSION = "2.2.2.1"
+end

data/test/commands_on_hashes_test.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+
+require File.expand_path("./helper", File.dirname(__FILE__))
+
+setup do
+  init Redis.new(OPTIONS)
+end
+
+load './test/lint/hashes.rb'
+
+test "Mapped HMGET in a pipeline returns plain array" do |r|
+  r.hset("foo", "f1", "s1")
+  r.hset("foo", "f2", "s2")
+
+  result = r.pipelined do
+    assert nil == r.mapped_hmget("foo", "f1", "f2")
+  end
+
+  assert result[0] == ["s1", "s2"]
+end

data/test/commands_on_lists_test.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+require File.expand_path("./helper", File.dirname(__FILE__))
+
+setup do
+  init Redis.new(OPTIONS)
+end
+
+load './test/lint/lists.rb'
+
+test "RPUSHX" do |r|
+  r.rpushx "foo", "s1"
+  r.rpush "foo", "s2"
+  r.rpushx "foo", "s3"
+
+  assert 2 == r.llen("foo")
+  assert ["s2", "s3"] == r.lrange("foo", 0, -1)
+end
+
+test "LPUSHX" do |r|
+  r.lpushx "foo", "s1"
+  r.lpush "foo", "s2"
+  r.lpushx "foo", "s3"
+
+  assert 2 == r.llen("foo")
+  assert ["s3", "s2"] == r.lrange("foo", 0, -1)
+end
+
+test "LINSERT" do |r|
+  r.rpush "foo", "s1"
+  r.rpush "foo", "s3"
+  r.linsert "foo", :before, "s3", "s2"
+
+  assert ["s1", "s2", "s3"] == r.lrange("foo", 0, -1)
+
+  assert_raise(RuntimeError) do
+    r.linsert "foo", :anywhere, "s3", "s2"
+  end
+end
+
+test "RPOPLPUSH" do |r|
+  r.rpush "foo", "s1"
+  r.rpush "foo", "s2"
+
+  assert "s2" == r.rpoplpush("foo", "bar")
+  assert ["s2"] == r.lrange("bar", 0, -1)
+  assert "s1" == r.rpoplpush("foo", "bar")
+  assert ["s1", "s2"] == r.lrange("bar", 0, -1)
+end
+
+test "BRPOPLPUSH" do |r|
+  r.rpush "foo", "s1"
+  r.rpush "foo", "s2"
+
+  assert_equal "s2", r.brpoplpush("foo", "bar", 1)
+
+  assert_equal nil, r.brpoplpush("baz", "qux", 1)
+
+  assert_equal ["s2"], r.lrange("bar", 0, -1)
+end