redis 0.1.2 → 0.2.0

data/Rakefile

@@ -5,10 +5,12 @@ require 'date'
 require 'spec/rake/spectask'
 require 'tasks/redis.tasks'
 
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+require 'redis'
 
 GEM = 'redis'
 GEM_NAME = 'redis'
-GEM_VERSION = '0.1.2'
+GEM_VERSION = RedisRb::VERSION
 AUTHORS = ['Ezra Zygmuntowicz', 'Taylor Weibley', 'Matthew Clark', 'Brian McKinney', 'Salvatore Sanfilippo', 'Luca Guidi']
 EMAIL = "ez@engineyard.com"
 HOMEPAGE = "http://github.com/ezmobius/redis-rb"

data/lib/edis.rb

@@ -0,0 +1,3 @@
+# This file allows for the running of redis-rb with a nice
+# command line look-and-feel: irb -rubygems -redis foo.rb
+require 'redis'

data/lib/redis/client.rb

@@ -0,0 +1,414 @@
+module RedisRb
+  class Client
+    OK      = "OK".freeze
+    MINUS    = "-".freeze
+    PLUS     = "+".freeze
+    COLON    = ":".freeze
+    DOLLAR   = "$".freeze
+    ASTERISK = "*".freeze
+
+    BULK_COMMANDS = {
+      "set"       => true,
+      "setnx"     => true,
+      "rpush"     => true,
+      "lpush"     => true,
+      "lset"      => true,
+      "lrem"      => true,
+      "sadd"      => true,
+      "srem"      => true,
+      "sismember" => true,
+      "echo"      => true,
+      "getset"    => true,
+      "smove"     => true,
+      "zadd"      => true,
+      "zincrby"   => true,
+      "zrem"      => true,
+      "zscore"    => true,
+      "zrank"     => true,
+      "zrevrank"  => true,
+      "hget"      => true,
+      "hdel"      => true,
+      "hexists"   => true
+    }
+
+    MULTI_BULK_COMMANDS = {
+      "mset"      => true,
+      "msetnx"    => true,
+      "hset"      => true
+    }
+
+    BOOLEAN_PROCESSOR = lambda{|r| r == 1 }
+
+    REPLY_PROCESSOR = {
+      "exists"    => BOOLEAN_PROCESSOR,
+      "sismember" => BOOLEAN_PROCESSOR,
+      "sadd"      => BOOLEAN_PROCESSOR,
+      "srem"      => BOOLEAN_PROCESSOR,
+      "smove"     => BOOLEAN_PROCESSOR,
+      "zadd"      => BOOLEAN_PROCESSOR,
+      "zrem"      => BOOLEAN_PROCESSOR,
+      "move"      => BOOLEAN_PROCESSOR,
+      "setnx"     => BOOLEAN_PROCESSOR,
+      "del"       => BOOLEAN_PROCESSOR,
+      "renamenx"  => BOOLEAN_PROCESSOR,
+      "expire"    => BOOLEAN_PROCESSOR,
+      "hset"      => BOOLEAN_PROCESSOR,
+      "hexists"   => BOOLEAN_PROCESSOR,
+      "info"      => lambda{|r|
+        info = {}
+        r.each_line {|kv|
+          k,v = kv.split(":",2).map{|x| x.chomp}
+          info[k.to_sym] = v
+        }
+        info
+      },
+      "keys"      => lambda{|r|
+        if r.is_a?(Array)
+            r
+        else
+            r.split(" ")
+        end
+      },
+      "hgetall"   => lambda{|r|
+        Hash[*r]
+      }
+    }
+
+    ALIASES = {
+      "flush_db"             => "flushdb",
+      "flush_all"            => "flushall",
+      "last_save"            => "lastsave",
+      "key?"                 => "exists",
+      "delete"               => "del",
+      "randkey"              => "randomkey",
+      "list_length"          => "llen",
+      "push_tail"            => "rpush",
+      "push_head"            => "lpush",
+      "pop_tail"             => "rpop",
+      "pop_head"             => "lpop",
+      "list_set"             => "lset",
+      "list_range"           => "lrange",
+      "list_trim"            => "ltrim",
+      "list_index"           => "lindex",
+      "list_rm"              => "lrem",
+      "set_add"              => "sadd",
+      "set_delete"           => "srem",
+      "set_count"            => "scard",
+      "set_member?"          => "sismember",
+      "set_members"          => "smembers",
+      "set_intersect"        => "sinter",
+      "set_intersect_store"  => "sinterstore",
+      "set_inter_store"      => "sinterstore",
+      "set_union"            => "sunion",
+      "set_union_store"      => "sunionstore",
+      "set_diff"             => "sdiff",
+      "set_diff_store"       => "sdiffstore",
+      "set_move"             => "smove",
+      "set_unless_exists"    => "setnx",
+      "rename_unless_exists" => "renamenx",
+      "type?"                => "type",
+      "zset_add"             => "zadd",
+      "zset_count"           => "zcard",
+      "zset_range_by_score"  => "zrangebyscore",
+      "zset_reverse_range"   => "zrevrange",
+      "zset_range"           => "zrange",
+      "zset_delete"          => "zrem",
+      "zset_score"           => "zscore",
+      "zset_incr_by"         => "zincrby",
+      "zset_increment_by"    => "zincrby"
+    }
+
+    DISABLED_COMMANDS = {
+      "monitor" => true,
+      "sync"    => true
+    }
+
+    def initialize(options = {})
+      @host    =  options[:host]    || '127.0.0.1'
+      @port    = (options[:port]    || 6379).to_i
+      @db      = (options[:db]      || 0).to_i
+      @timeout = (options[:timeout] || 5).to_i
+      @password = options[:password]
+      @logger  =  options[:logger]
+      @thread_safe = options[:thread_safe]
+      @binary_keys = options[:binary_keys]
+      @mutex = Mutex.new if @thread_safe
+      @sock = nil
+
+      @logger.info { self.to_s } if @logger
+    end
+
+    def to_s
+      "Redis Client connected to #{server} against DB #{@db}"
+    end
+
+    def server
+      "#{@host}:#{@port}"
+    end
+
+    def connect_to_server
+      @sock = connect_to(@host, @port, @timeout == 0 ? nil : @timeout)
+      call_command(["auth",@password]) if @password
+      call_command(["select",@db]) unless @db == 0
+    end
+
+    def connect_to(host, port, timeout=nil)
+      # We support connect() timeout only if system_timer is availabe
+      # or if we are running against Ruby >= 1.9
+      # Timeout reading from the socket instead will be supported anyway.
+      if @timeout != 0 and RedisTimer
+        begin
+          sock = TCPSocket.new(host, port)
+        rescue Timeout::Error
+          @sock = nil
+          raise Timeout::Error, "Timeout connecting to the server"
+        end
+      else
+        sock = TCPSocket.new(host, port)
+      end
+      sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+      # If the timeout is set we set the low level socket options in order
+      # to make sure a blocking read will return after the specified number
+      # of seconds. This hack is from memcached ruby client.
+      if timeout
+        secs   = Integer(timeout)
+        usecs  = Integer((timeout - secs) * 1_000_000)
+        optval = [secs, usecs].pack("l_2")
+        begin
+          sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+          sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+        rescue Exception => ex
+          # Solaris, for one, does not like/support socket timeouts.
+          @logger.info "Unable to use raw socket timeouts: #{ex.class.name}: #{ex.message}" if @logger
+        end
+      end
+      sock
+    end
+
+    def method_missing(*argv)
+      call_command(argv)
+    end
+
+    def call_command(argv)
+      @logger.debug { argv.inspect } if @logger
+
+      # this wrapper to raw_call_command handle reconnection on socket
+      # error. We try to reconnect just one time, otherwise let the error
+      # araise.
+      connect_to_server if !@sock
+
+      begin
+        raw_call_command(argv.dup)
+      rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED
+        @sock.close rescue nil
+        @sock = nil
+        connect_to_server
+        raw_call_command(argv.dup)
+      end
+    end
+
+    def raw_call_command(argvp)
+      if argvp[0].is_a?(Array)
+        argvv = argvp
+        pipeline = true
+      else
+        argvv = [argvp]
+      end
+
+      if @binary_keys or MULTI_BULK_COMMANDS[argvv[0][0].to_s]
+        command = ""
+        argvv.each do |argv|
+          command << "*#{argv.size}\r\n"
+          argv.each{|a|
+            a = a.to_s
+            command << "$#{get_size(a)}\r\n"
+            command << a
+            command << "\r\n"
+          }
+        end
+      else
+        command = ""
+        argvv.each do |argv|
+          bulk = nil
+          argv[0] = argv[0].to_s.downcase
+          argv[0] = ALIASES[argv[0]] if ALIASES[argv[0]]
+          raise "#{argv[0]} command is disabled" if DISABLED_COMMANDS[argv[0]]
+          if BULK_COMMANDS[argv[0]] and argv.length > 1
+            bulk = argv[-1].to_s
+            argv[-1] = get_size(bulk)
+          end
+          command << "#{argv.join(' ')}\r\n"
+          command << "#{bulk}\r\n" if bulk
+        end
+      end
+      results = maybe_lock { process_command(command, argvv) }
+
+      return pipeline ? results : results[0]
+    end
+
+    def process_command(command, argvv)
+      @sock.write(command)
+      argvv.map do |argv|
+        processor = REPLY_PROCESSOR[argv[0].to_s]
+        processor ? processor.call(read_reply) : read_reply
+      end
+    end
+
+    def maybe_lock(&block)
+      if @thread_safe
+        @mutex.synchronize(&block)
+      else
+        block.call
+      end
+    end
+
+    def select(*args)
+      raise "SELECT not allowed, use the :db option when creating the object"
+    end
+
+    def [](key)
+      self.get(key)
+    end
+
+    def []=(key,value)
+      set(key,value)
+    end
+
+    def set(key, value, expiry=nil)
+      s = call_command([:set, key, value]) == OK
+      expire(key, expiry) if s && expiry
+      s
+    end
+
+    def mset(*args)
+      hsh = args.pop if Hash === args.last
+      if hsh
+        call_command(hsh.to_a.flatten.unshift(:mset))
+      else
+        call_command(args.unshift(:mset))
+      end
+    end
+
+    def msetnx(*args)
+      hsh = args.pop if Hash === args.last
+      if hsh
+        call_command(hsh.to_a.flatten.unshift(:msetnx))
+      else
+        call_command(args.unshift(:msetnx))
+      end
+    end
+
+    def sort(key, options = {})
+      cmd = ["SORT"]
+      cmd << key
+      cmd << "BY #{options[:by]}" if options[:by]
+      cmd << "GET #{[options[:get]].flatten * ' GET '}" if options[:get]
+      cmd << "#{options[:order]}" if options[:order]
+      cmd << "LIMIT #{options[:limit].join(' ')}" if options[:limit]
+      cmd << "STORE #{options[:store]}" if options[:store]
+      call_command(cmd)
+    end
+
+    def incr(key, increment = nil)
+      call_command(increment ? ["incrby",key,increment] : ["incr",key])
+    end
+
+    def decr(key,decrement = nil)
+      call_command(decrement ? ["decrby",key,decrement] : ["decr",key])
+    end
+
+    # Similar to memcache.rb's #get_multi, returns a hash mapping
+    # keys to values.
+    def mapped_mget(*keys)
+      result = {}
+      mget(*keys).each do |value|
+        key = keys.shift
+        result.merge!(key => value) unless value.nil?
+      end
+      result
+    end
+
+    # Ruby defines a now deprecated type method so we need to override it here
+    # since it will never hit method_missing
+    def type(key)
+      call_command(['type', key])
+    end
+
+    def quit
+      call_command(['quit'])
+    rescue Errno::ECONNRESET
+    end
+
+    def pipelined(&block)
+      pipeline = Pipeline.new self
+      yield pipeline
+      pipeline.execute
+    end
+
+    def read_reply
+      # We read the first byte using read() mainly because gets() is
+      # immune to raw socket timeouts.
+      begin
+        rtype = @sock.read(1)
+      rescue Errno::EAGAIN
+        # We want to make sure it reconnects on the next command after the
+        # timeout. Otherwise the server may reply in the meantime leaving
+        # the protocol in a desync status.
+        @sock = nil
+        raise Errno::EAGAIN, "Timeout reading from the socket"
+      end
+
+      raise Errno::ECONNRESET,"Connection lost" if !rtype
+      line = @sock.gets
+      case rtype
+      when MINUS
+        raise MINUS + line.strip
+      when PLUS
+        line.strip
+      when COLON
+        line.to_i
+      when DOLLAR
+        bulklen = line.to_i
+        return nil if bulklen == -1
+        data = @sock.read(bulklen)
+        @sock.read(2) # CRLF
+        data
+      when ASTERISK
+        objects = line.to_i
+        return nil if bulklen == -1
+        res = []
+        objects.times {
+          res << read_reply
+        }
+        res
+      else
+        raise "Protocol error, got '#{rtype}' as initial reply byte"
+      end
+    end
+
+    def exec
+      # Need to override Kernel#exec.
+      call_command([:exec])
+    end
+
+    def multi(&block)
+      result = call_command [:multi]
+
+      return result unless block_given?
+
+      begin
+        yield(self)
+        exec
+      rescue Exception => e
+        discard
+        raise e
+      end
+    end
+
+    private
+      def get_size(string)
+        string.respond_to?(:bytesize) ? string.bytesize : string.size
+      end
+  end
+end

data/lib/redis/dist_redis.rb

@@ -0,0 +1,114 @@
+require 'redis/hash_ring'
+
+module RedisRb
+  class DistRedis
+    attr_reader :ring
+    def initialize(opts={})
+      hosts = []
+
+      db = opts[:db] || nil
+      timeout = opts[:timeout] || nil
+
+      raise "No hosts given" unless opts[:hosts]
+
+      opts[:hosts].each do |h|
+        host, port = h.split(':')
+        hosts << Client.new(:host => host, :port => port, :db => db, :timeout => timeout)
+      end
+
+      @ring = HashRing.new hosts
+    end
+
+    def node_for_key(key)
+      key = $1 if key =~ /\{(.*)?\}/
+      @ring.get_node(key)
+    end
+
+    def add_server(server)
+      server, port = server.split(':')
+      @ring.add_node Client.new(:host => server, :port => port)
+    end
+
+    def method_missing(sym, *args, &blk)
+      if redis = node_for_key(args.first.to_s)
+        redis.send sym, *args, &blk
+      else
+        super
+      end
+    end
+
+    def node_keys(glob)
+      @ring.nodes.map do |red|
+        red.keys(glob)
+      end
+    end
+
+    def keys(glob)
+      node_keys(glob).flatten
+    end
+
+    def save
+      on_each_node :save
+    end
+
+    def bgsave
+      on_each_node :bgsave
+    end
+
+    def quit
+      on_each_node :quit
+    end
+
+    def flush_all
+      on_each_node :flush_all
+    end
+    alias_method :flushall, :flush_all
+
+    def flush_db
+      on_each_node :flush_db
+    end
+    alias_method :flushdb, :flush_db
+
+    def delete_cloud!
+      @ring.nodes.each do |red|
+        red.keys("*").each do |key|
+          red.delete key
+        end
+      end
+    end
+
+    def on_each_node(command, *args)
+      @ring.nodes.each do |red|
+        red.send(command, *args)
+      end
+    end
+
+    def mset()
+
+    end
+
+    def mget(*keyz)
+      results = {}
+      kbn = keys_by_node(keyz)
+      kbn.each do |node, node_keyz|
+        node.mapped_mget(*node_keyz).each do |k, v|
+          results[k] = v
+        end
+      end
+      keyz.flatten.map { |k| results[k] }
+    end
+
+    def keys_by_node(*keyz)
+      keyz.flatten.inject({}) do |kbn, k|
+        node = node_for_key(k)
+        next if kbn[node] && kbn[node].include?(k)
+        kbn[node] ||= []
+        kbn[node] << k
+        kbn
+      end
+    end
+  end
+end
+
+# For backwards compatibility
+DistRedis = RedisRb::DistRedis

data/lib/redis/hash_ring.rb

@@ -0,0 +1,131 @@
+require 'zlib'
+
+module RedisRb
+  class HashRing
+
+    POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+    attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+    # nodes is a list of objects that have a proper to_s representation.
+    # replicas indicates how many virtual points should be used pr. node,
+    # replicas are required to improve the distribution.
+    def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+      @replicas = replicas
+      @ring = {}
+      @nodes = []
+      @sorted_keys = []
+      nodes.each do |node|
+        add_node(node)
+      end
+    end
+
+    # Adds a `node` to the hash ring (including a number of replicas).
+    def add_node(node)
+      @nodes << node
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node}:#{i}")
+        @ring[key] = node
+        @sorted_keys << key
+      end
+      @sorted_keys.sort!
+    end
+
+    def remove_node(node)
+      @nodes.reject!{|n| n.to_s == node.to_s}
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node}:#{i}")
+        @ring.delete(key)
+        @sorted_keys.reject! {|k| k == key}
+      end
+    end
+
+    # get the node in the hash ring for this key
+    def get_node(key)
+      get_node_pos(key)[0]
+    end
+
+    def get_node_pos(key)
+      return [nil,nil] if @ring.size == 0
+      crc = Zlib.crc32(key)
+      idx = HashRing.binary_search(@sorted_keys, crc)
+      return [@ring[@sorted_keys[idx]], idx]
+    end
+
+    def iter_nodes(key)
+      return [nil,nil] if @ring.size == 0
+      node, pos = get_node_pos(key)
+      @sorted_keys[pos..-1].each do |k|
+        yield @ring[k]
+      end
+    end
+
+    class << self
+
+      # gem install RubyInline to use this code
+      # Native extension to perform the binary search within the hashring.
+      # There's a pure ruby version below so this is purely optional
+      # for performance.  In testing 20k gets and sets, the native
+      # binary search shaved about 12% off the runtime (9sec -> 8sec).
+      begin
+        require 'inline'
+        inline do |builder|
+          builder.c <<-EOM
+          int binary_search(VALUE ary, unsigned int r) {
+              int upper = RARRAY_LEN(ary) - 1;
+              int lower = 0;
+              int idx = 0;
+
+              while (lower <= upper) {
+                  idx = (lower + upper) / 2;
+
+                  VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                  unsigned int l = NUM2UINT(continuumValue);
+                  if (l == r) {
+                      return idx;
+                  }
+                  else if (l > r) {
+                      upper = idx - 1;
+                  }
+                  else {
+                      lower = idx + 1;
+                  }
+              }
+              if (upper < 0) {
+                upper = RARRAY_LEN(ary) - 1;
+              }
+              return upper;
+          }
+          EOM
+        end
+      rescue Exception => e
+        # Find the closest index in HashRing with value <= the given value
+        def binary_search(ary, value, &block)
+          upper = ary.size - 1
+          lower = 0
+          idx = 0
+
+          while(lower <= upper) do
+            idx = (lower + upper) / 2
+            comp = ary[idx] <=> value
+
+            if comp == 0
+              return idx
+            elsif comp > 0
+              upper = idx - 1
+            else
+              lower = idx + 1
+            end
+          end
+
+          if upper < 0
+            upper = ary.size - 1
+          end
+          return upper
+        end
+
+      end
+    end
+
+  end
+end

data/lib/{pipeline.rb → redis/pipeline.rb}

@@ -1,12 +1,12 @@
-class Redis
-  class Pipeline < Redis
+module RedisRb
+  class Pipeline < Client
     BUFFER_SIZE = 50_000
-    
+
     def initialize(redis)
       @redis = redis
       @commands = []
     end
-   
+
     def call_command(command)
       @commands << command
     end