kali-redis 0.1.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Ezra Zygmuntowicz
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,36 @@
+# redis-rb
+
+A ruby client library for the redis key value storage system.
+
+## Information about redis
+
+Redis is a key value store with some interesting features:
+1. It's fast.
+2. Keys are strings but values can have types of "NONE", "STRING", "LIST",  or "SET".  List's can be atomically push'd, pop'd, lpush'd, lpop'd and indexed.  This allows you to store things like lists of comments under one key while retaining the ability to append comments without reading and putting back the whole list.
+
+See [redis on code.google.com](http://code.google.com/p/redis/wiki/README) for more information.
+
+See the build on [RunCodeRun](http://runcoderun.com/rsanheim/redis-rb)
+
+## Dependencies
+
+1. rspec - 
+		sudo gem install rspec
+
+2. redis - 
+
+		rake redis:install
+
+2. dtach - 
+
+		rake dtach:install
+
+3. git - git is the new black.
+
+## Setup
+
+Use the tasks mentioned above (in Dependencies) to get your machine setup.
+
+## Examples
+
+Check the examples/ directory.  *Note* you need to have redis-server running first.

data/Rakefile

@@ -0,0 +1,62 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require 'spec/rake/spectask'
+require 'tasks/redis.tasks'
+
+
+GEM = 'kali-redis'
+GEM_NAME = 'kali'
+GEM_VERSION = '0.1.1'
+AUTHORS = ['Ezra Zygmuntowicz', 'Taylor Weibley', 'Matthew Clark', 'Brian McKinney', 'Salvatore Sanfilippo', 'Luca Guidi', 'Mathieu Poumeyrol']
+EMAIL = "ez@engineyard.com"
+HOMEPAGE = "http://github.com/kali/redis-rb"
+SUMMARY = "Ruby client library for redis key value storage server"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["LICENSE"]
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.authors = AUTHORS
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.add_development_dependency "rspec"
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.markdown Rakefile) + Dir.glob("{lib,tasks,spec}/**/*")
+end
+
+task :default => :spec
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = %w(-fs --color)
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end
+
+desc "Run all examples with RCov"
+Spec::Rake::SpecTask.new(:rcov) do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.rcov = true
+end

data/lib/dist_redis.rb

@@ -0,0 +1,124 @@
+require 'redis'
+require 'hash_ring'
+class DistRedis
+  attr_reader :ring
+  def initialize(opts={})
+    hosts = []
+
+    db = opts[:db] || nil
+    timeout = opts[:timeout] || nil
+
+    raise Error, "No hosts given" unless opts[:hosts]
+
+    opts[:hosts].each do |h|
+      host, port = h.split(':')
+      hosts << Redis.new(:host => host, :port => port, :db => db, :timeout => timeout)
+    end
+
+    @ring = HashRing.new hosts
+  end
+
+  def node_for_key(key)
+    key = $1 if key =~ /\{(.*)?\}/
+    @ring.get_node(key)
+  end
+
+  def add_server(server)
+    server, port = server.split(':')
+    @ring.add_node Redis.new(:host => server, :port => port)
+  end
+
+  def method_missing(sym, *args, &blk)
+    if redis = node_for_key(args.first.to_s)
+      redis.send sym, *args, &blk
+    else
+      super
+    end
+  end
+
+  def keys(glob)
+    @ring.nodes.map do |red|
+      red.keys(glob)
+    end
+  end
+
+  def save
+    on_each_node :save
+  end
+
+  def bgsave
+    on_each_node :bgsave
+  end
+
+  def quit
+    on_each_node :quit
+  end
+
+  def flush_all
+    on_each_node :flush_all
+  end
+  alias_method :flushall, :flush_all
+
+  def flush_db
+    on_each_node :flush_db
+  end
+  alias_method :flushdb, :flush_db
+
+  def delete_cloud!
+    @ring.nodes.each do |red|
+      red.keys("*").each do |key|
+        red.delete key
+      end
+    end
+  end
+
+  def on_each_node(command, *args)
+    @ring.nodes.each do |red|
+      red.send(command, *args)
+    end
+  end
+
+end
+
+
+if __FILE__ == $0
+
+r = DistRedis.new 'localhost:6379', 'localhost:6380', 'localhost:6381', 'localhost:6382'
+  r['urmom'] = 'urmom'
+  r['urdad'] = 'urdad'
+  r['urmom1'] = 'urmom1'
+  r['urdad1'] = 'urdad1'
+  r['urmom2'] = 'urmom2'
+  r['urdad2'] = 'urdad2'
+  r['urmom3'] = 'urmom3'
+  r['urdad3'] = 'urdad3'
+  p r['urmom']
+  p r['urdad']
+  p r['urmom1']
+  p r['urdad1']
+  p r['urmom2']
+  p r['urdad2']
+  p r['urmom3']
+  p r['urdad3']
+
+  r.push_tail 'listor', 'foo1'
+  r.push_tail 'listor', 'foo2'
+  r.push_tail 'listor', 'foo3'
+  r.push_tail 'listor', 'foo4'
+  r.push_tail 'listor', 'foo5'
+
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+
+  puts "key distribution:"
+
+  r.ring.nodes.each do |red|
+    p [red.port, red.keys("*")]
+  end
+  r.delete_cloud!
+  p r.keys('*')
+
+end

data/lib/hash_ring.rb

@@ -0,0 +1,135 @@
+require 'zlib'
+
+class HashRing
+
+  POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+  attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+  # nodes is a list of objects that have a proper to_s representation.
+  # replicas indicates how many virtual points should be used pr. node,
+  # replicas are required to improve the distribution.
+  def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+    @replicas = replicas
+    @ring = {}
+    @nodes = []
+    @sorted_keys = []
+    nodes.each do |node|
+      add_node(node)
+    end
+  end
+  
+  # Adds a `node` to the hash ring (including a number of replicas).
+  def add_node(node)
+    @nodes << node
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{i}")
+      @ring[key] = node
+      @sorted_keys << key
+    end
+    @sorted_keys.sort!
+  end
+  
+  def remove_node(node)
+    @nodes.reject!{|n| n.to_s == node.to_s}
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{i}")
+      @ring.delete(key)
+      @sorted_keys.reject! {|k| k == key}
+    end
+  end
+  
+  # get the node in the hash ring for this key
+  def get_node(key)
+    get_node_pos(key)[0]
+  end
+  
+  def get_node_pos(key)
+    return [nil,nil] if @ring.size == 0
+    crc = Zlib.crc32(key)
+    idx = HashRing.binary_search(@sorted_keys, crc)
+    return [@ring[@sorted_keys[idx]], idx]
+  end
+  
+  def iter_nodes(key)
+    return [nil,nil] if @ring.size == 0
+    node, pos = get_node_pos(key)
+    @sorted_keys[pos..-1].each do |k|
+      yield @ring[k]
+    end  
+  end
+  
+  class << self
+
+    # gem install RubyInline to use this code
+    # Native extension to perform the binary search within the hashring.
+    # There's a pure ruby version below so this is purely optional
+    # for performance.  In testing 20k gets and sets, the native
+    # binary search shaved about 12% off the runtime (9sec -> 8sec).
+    begin
+      require 'inline'
+      inline do |builder|
+        builder.c <<-EOM
+        int binary_search(VALUE ary, unsigned int r) {
+            int upper = RARRAY_LEN(ary) - 1;
+            int lower = 0;
+            int idx = 0;
+
+            while (lower <= upper) {
+                idx = (lower + upper) / 2;
+
+                VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                unsigned int l = NUM2UINT(continuumValue);
+                if (l == r) {
+                    return idx;
+                }
+                else if (l > r) {
+                    upper = idx - 1;
+                }
+                else {
+                    lower = idx + 1;
+                }
+            }
+            if (upper < 0) {
+              upper = RARRAY_LEN(ary) - 1;
+            }
+            return upper;
+        }
+        EOM
+      end
+    rescue Exception => e
+      # Find the closest index in HashRing with value <= the given value
+      def binary_search(ary, value, &block)
+        upper = ary.size - 1
+        lower = 0
+        idx = 0
+
+        while(lower <= upper) do
+          idx = (lower + upper) / 2
+          comp = ary[idx] <=> value
+
+          if comp == 0
+            return idx
+          elsif comp > 0
+            upper = idx - 1
+          else
+            lower = idx + 1
+          end
+        end
+        
+        if upper < 0
+          upper = ary.size - 1
+        end
+        return upper
+      end
+
+    end
+  end
+
+end
+
+# ring = HashRing.new ['server1', 'server2', 'server3']
+# p ring
+# #
+# p ring.get_node "kjhjkjlkjlkkh"
+# 

data/lib/pipeline.rb

@@ -0,0 +1,21 @@
+class Redis
+  class Pipeline < Redis
+    BUFFER_SIZE = 50_000
+    
+    def initialize(redis)
+      @redis = redis
+      @commands = []
+    end
+   
+    def call_command(command)
+      @commands << command
+    end
+
+    def execute
+      return if @commands.empty?
+      @redis.call_command(@commands)
+      @commands.clear
+    end
+
+  end
+end

data/lib/redis.rb

@@ -0,0 +1,373 @@
+require 'socket'
+require File.join(File.dirname(__FILE__),'pipeline')
+
+begin
+  if RUBY_VERSION >= '1.9'
+    require 'timeout'
+    RedisTimer = Timeout
+  else
+    require 'system_timer'
+    RedisTimer = SystemTimer
+  end
+rescue LoadError
+  RedisTimer = nil
+end
+
+class Redis
+  OK      = "OK".freeze
+  MINUS    = "-".freeze
+  PLUS     = "+".freeze
+  COLON    = ":".freeze
+  DOLLAR   = "$".freeze
+  ASTERISK = "*".freeze
+
+  BULK_COMMANDS = {
+    "set"       => true,
+    "setnx"     => true,
+    "rpush"     => true,
+    "lpush"     => true,
+    "lset"      => true,
+    "lrem"      => true,
+    "sadd"      => true,
+    "srem"      => true,
+    "sismember" => true,
+    "rpoplpush" => true,
+    "echo"      => true,
+    "getset"    => true,
+    "smove"     => true,
+    "zadd"      => true,
+    "zincrby"   => true,
+    "zrem"      => true,
+    "zscore"    => true
+  }
+  
+  MULTI_BULK_COMMANDS = {
+    "mset"      => true,
+    "msetnx"    => true,
+    "zhadd"     => true,
+  }
+
+  BOOLEAN_PROCESSOR = lambda{|r| r == 1 }
+
+  REPLY_PROCESSOR = {
+    "exists"    => BOOLEAN_PROCESSOR,
+    "sismember" => BOOLEAN_PROCESSOR,
+    "sadd"      => BOOLEAN_PROCESSOR,
+    "srem"      => BOOLEAN_PROCESSOR,
+    "smove"     => BOOLEAN_PROCESSOR,
+    "zadd"      => BOOLEAN_PROCESSOR,
+    "zrem"      => BOOLEAN_PROCESSOR,
+    "move"      => BOOLEAN_PROCESSOR,
+    "setnx"     => BOOLEAN_PROCESSOR,
+    "del"       => BOOLEAN_PROCESSOR,
+    "renamenx"  => BOOLEAN_PROCESSOR,
+    "expire"    => BOOLEAN_PROCESSOR,
+    "keys"      => lambda{|r| r.split(" ")},
+    "info"      => lambda{|r|
+      info = {}
+      r.each_line {|kv|
+        k,v = kv.split(":",2).map{|x| x.chomp}
+        info[k.to_sym] = v
+      }
+      info
+    }
+  }
+
+  ALIASES = {
+    "flush_db"             => "flushdb",
+    "flush_all"            => "flushall",
+    "last_save"            => "lastsave",
+    "key?"                 => "exists",
+    "delete"               => "del",
+    "randkey"              => "randomkey",
+    "list_length"          => "llen",
+    "push_tail"            => "rpush",
+    "push_head"            => "lpush",
+    "pop_tail"             => "rpop",
+    "pop_head"             => "lpop",
+    "list_set"             => "lset",
+    "list_range"           => "lrange",
+    "list_trim"            => "ltrim",
+    "list_index"           => "lindex",
+    "list_rm"              => "lrem",
+    "set_add"              => "sadd",
+    "set_delete"           => "srem",
+    "set_count"            => "scard",
+    "set_member?"          => "sismember",
+    "set_members"          => "smembers",
+    "set_intersect"        => "sinter",
+    "set_intersect_store"  => "sinterstore",
+    "set_inter_store"      => "sinterstore",
+    "set_union"            => "sunion",
+    "set_union_store"      => "sunionstore",
+    "set_diff"             => "sdiff",
+    "set_diff_store"       => "sdiffstore",
+    "set_move"             => "smove",
+    "set_unless_exists"    => "setnx",
+    "rename_unless_exists" => "renamenx",
+    "type?"                => "type",
+    "zset_add"             => "zadd",
+    "zset_count"           => "zcard",
+    "zset_range_by_score"  => "zrangebyscore",
+    "zset_reverse_range"   => "zrevrange",
+    "zset_range"           => "zrange",
+    "zset_delete"          => "zrem",
+    "zset_score"           => "zscore",
+    "zset_incr_by"         => "zincrby",
+    "zset_increment_by"    => "zincrby"
+  }
+
+  DISABLED_COMMANDS = {
+    "monitor" => true,
+    "sync"    => true
+  }
+
+  def initialize(options = {})
+    @host    =  options[:host]    || '127.0.0.1'
+    @port    = (options[:port]    || 6379).to_i
+    @db      = (options[:db]      || 0).to_i
+    @timeout = (options[:timeout] || 5).to_i
+    @password = options[:password]
+    @logger  =  options[:logger]
+    @thread_safe = options[:thread_safe]
+    @mutex = Mutex.new if @thread_safe
+    @sock = nil
+
+    @logger.info { self.to_s } if @logger
+  end
+
+  def to_s
+    "Redis Client connected to #{server} against DB #{@db}"
+  end
+
+  def server
+    "#{@host}:#{@port}"
+  end
+
+  def connect_to_server
+    @sock = connect_to(@host, @port, @timeout == 0 ? nil : @timeout)
+    call_command(["auth",@password]) if @password
+    call_command(["select",@db]) unless @db == 0
+  end
+
+  def connect_to(host, port, timeout=nil)
+    # We support connect() timeout only if system_timer is availabe
+    # or if we are running against Ruby >= 1.9
+    # Timeout reading from the socket instead will be supported anyway.
+    if @timeout != 0 and RedisTimer
+      begin
+        sock = TCPSocket.new(host, port)
+      rescue Timeout::Error
+        @sock = nil
+        raise Timeout::Error, "Timeout connecting to the server"
+      end
+    else
+      sock = TCPSocket.new(host, port)
+    end
+    sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+    # If the timeout is set we set the low level socket options in order
+    # to make sure a blocking read will return after the specified number
+    # of seconds. This hack is from memcached ruby client.
+    if timeout
+      secs   = Integer(timeout)
+      usecs  = Integer((timeout - secs) * 1_000_000)
+      optval = [secs, usecs].pack("l_2")
+      begin
+        sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+        sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+      rescue Exception => ex
+        # Solaris, for one, does not like/support socket timeouts.
+        @logger.info "Unable to use raw socket timeouts: #{ex.class.name}: #{ex.message}" if @logger
+      end
+    end
+    sock
+  end
+
+  def method_missing(*argv)
+    call_command(argv)
+  end
+
+  def call_command(argv)
+    @logger.debug { argv.inspect } if @logger
+
+    # this wrapper to raw_call_command handle reconnection on socket
+    # error. We try to reconnect just one time, otherwise let the error
+    # araise.
+    connect_to_server if !@sock
+
+    begin
+      raw_call_command(argv.dup)
+    rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED
+      @sock.close rescue nil
+      @sock = nil
+      connect_to_server
+      raw_call_command(argv.dup)
+    end
+  end
+
+  def raw_call_command(argvp)
+    pipeline = argvp[0].is_a?(Array)
+
+    unless pipeline
+      argvv = [argvp]
+    else
+      argvv = argvp
+    end
+
+    if MULTI_BULK_COMMANDS[argvv.flatten[0].to_s]
+      # TODO improve this code
+      argvp   = argvv.flatten
+      command = ["*#{argvp.size}"]
+      argvp.each do |v|
+        v = v.to_s
+        command << "$#{get_size(v)}"
+        command << v
+      end
+      command = command.map {|cmd| "#{cmd}\r\n"}.join
+    else
+      command = ""
+      argvv.each do |argv|
+        bulk = nil
+        argv[0] = argv[0].to_s.downcase
+        argv[0] = ALIASES[argv[0]] if ALIASES[argv[0]]
+        raise "#{argv[0]} command is disabled" if DISABLED_COMMANDS[argv[0]]
+        if BULK_COMMANDS[argv[0]] and argv.length > 1
+          bulk = argv[-1].to_s
+          argv[-1] = get_size(bulk)
+        end
+        command << "#{argv.join(' ')}\r\n"
+        command << "#{bulk}\r\n" if bulk
+      end
+    end
+    results = maybe_lock { process_command(command, argvv) }
+    
+    return pipeline ? results : results[0]
+  end
+
+  def process_command(command, argvv)
+    @sock.write(command)
+    argvv.map do |argv|
+      processor = REPLY_PROCESSOR[argv[0]]
+      processor ? processor.call(read_reply) : read_reply
+    end
+  end
+  
+  def maybe_lock(&block)
+    if @thread_safe
+      @mutex.synchronize &block
+    else
+      block.call
+    end
+  end
+
+  def select(*args)
+    raise "SELECT not allowed, use the :db option when creating the object"
+  end
+
+  def [](key)
+    self.get(key)
+  end
+
+  def []=(key,value)
+    set(key,value)
+  end
+
+  def set(key, value, expiry=nil)
+    s = call_command([:set, key, value]) == OK
+    expire(key, expiry) if s && expiry
+    s
+  end
+
+  def sort(key, options = {})
+    cmd = ["SORT"]
+    cmd << key
+    cmd << "BY #{options[:by]}" if options[:by]
+    cmd << "GET #{[options[:get]].flatten * ' GET '}" if options[:get]
+    cmd << "#{options[:order]}" if options[:order]
+    cmd << "LIMIT #{options[:limit].join(' ')}" if options[:limit]
+    call_command(cmd)
+  end
+
+  def incr(key, increment = nil)
+    call_command(increment ? ["incrby",key,increment] : ["incr",key])
+  end
+
+  def decr(key,decrement = nil)
+    call_command(decrement ? ["decrby",key,decrement] : ["decr",key])
+  end
+
+  # Similar to memcache.rb's #get_multi, returns a hash mapping
+  # keys to values.
+  def mapped_mget(*keys)
+    result = {}
+    mget(*keys).each do |value|
+      key = keys.shift
+      result.merge!(key => value) unless value.nil?
+    end
+    result
+  end
+
+  # Ruby defines a now deprecated type method so we need to override it here
+  # since it will never hit method_missing
+  def type(key)
+    call_command(['type', key])
+  end
+
+  def quit
+    call_command(['quit'])
+  rescue Errno::ECONNRESET
+  end
+
+  def pipelined(&block)
+    pipeline = Pipeline.new self
+    yield pipeline
+    pipeline.execute
+  end
+
+  def read_reply
+    # We read the first byte using read() mainly because gets() is
+    # immune to raw socket timeouts.
+    begin
+      rtype = @sock.read(1)
+    rescue Errno::EAGAIN
+      # We want to make sure it reconnects on the next command after the
+      # timeout. Otherwise the server may reply in the meantime leaving
+      # the protocol in a desync status.
+      @sock = nil
+      raise Errno::EAGAIN, "Timeout reading from the socket"
+    end
+
+    raise Errno::ECONNRESET,"Connection lost" if !rtype
+    line = @sock.gets
+    case rtype
+    when MINUS
+      raise MINUS + line.strip
+    when PLUS
+      line.strip
+    when COLON
+      line.to_i
+    when DOLLAR
+      bulklen = line.to_i
+      return nil if bulklen == -1
+      data = @sock.read(bulklen)
+      @sock.read(2) # CRLF
+      data
+    when ASTERISK
+      objects = line.to_i
+      return nil if bulklen == -1
+      res = []
+      objects.times {
+        res << read_reply
+      }
+      res
+    else
+      raise "Protocol error, got '#{rtype}' as initial reply byte"
+    end
+  end
+
+  private
+    def get_size(string)
+      string.respond_to?(:bytesize) ? string.bytesize : string.size
+    end
+end