timcharper-redis 0.0.3.4

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Ezra Zygmuntowicz
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,31 @@
+# redis-rb
+
+A ruby client library for the redis key value storage system.
+
+## Information about redis
+
+Redis is a key value store with some interesting features:
+1. It's fast.
+2. Keys are strings but values can have types of "NONE", "STRING", "LIST",  or "SET".  List's can be atomically push'd, pop'd, lpush'd, lpop'd and indexed.  This allows you to store things like lists of comments under one key while retaining the ability to append comments without reading and putting back the whole list.
+
+See [redis on code.google.com](http://code.google.com/p/redis/wiki/README) for more information.
+
+## Dependencies
+
+1. redis - 
+
+		rake redis:install
+
+2. dtach - 
+
+		rake dtach:install
+
+3. svn - git is the new black, but we need it for the google codes.
+
+## Setup
+
+Use the tasks mentioned above (in Dependencies) to get your machine setup.
+
+## Examples
+
+Check the examples/ directory.  *Note* you need to have redis-server running first.

data/Rakefile

@@ -0,0 +1,59 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require 'spec/rake/spectask'
+require 'tasks/redis.tasks'
+
+
+GEM = 'redis'
+GEM_NAME = 'redis'
+GEM_VERSION = '0.0.3.3'
+AUTHORS = ['Ezra Zygmuntowicz', 'Taylor Weibley', 'Matthew Clark']
+EMAIL = "matt.clark@punchstock.com"
+HOMEPAGE = "http://github.com/winescout/redis-rb"
+SUMMARY = "Ruby client library for redis key value storage server"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["LICENSE"]
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.authors = AUTHORS
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  
+  # Uncomment this to add a dependency
+  # s.add_dependency "foo"
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.markdown Rakefile) + Dir.glob("{lib,spec}/**/*")
+end
+
+task :default => :spec
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = %w(-fs --color)
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end

data/lib/dist_redis.rb

@@ -0,0 +1,111 @@
+require 'redis'
+require 'hash_ring'
+class DistRedis
+  attr_reader :ring
+  def initialize(*servers)
+    srvs = []
+    servers.each do |s|
+      server, port = s.split(':')
+      srvs << Redis.new(:host => server, :port => port)
+    end
+    @ring = HashRing.new srvs
+  end
+  
+  def node_for_key(key)
+    if key =~ /\{(.*)?\}/
+      key = $1
+    end
+    @ring.get_node(key)
+  end
+  
+  def add_server(server)
+    server, port = server.split(':')
+    @ring.add_node Redis.new(:host => server, :port => port)
+  end
+  
+  def method_missing(sym, *args, &blk)
+    if redis = node_for_key(args.first.to_s)
+      redis.send sym, *args, &blk
+    else
+      super
+    end
+  end
+  
+  def keys(glob)
+    keyz = []
+    @ring.nodes.each do |red|
+      keyz.concat red.keys(glob)
+    end
+    keyz
+  end
+  
+  def save
+    @ring.nodes.each do |red|
+      red.save
+    end
+  end
+  
+  def bgsave
+    @ring.nodes.each do |red|
+      red.bgsave
+    end
+  end
+  
+  def quit
+    @ring.nodes.each do |red|
+      red.quit
+    end
+  end
+  
+  def delete_cloud!
+    @ring.nodes.each do |red|
+      red.keys("*").each do |key|
+        red.delete key
+      end  
+    end
+  end
+  
+end
+
+
+if __FILE__ == $0
+
+r = DistRedis.new 'localhost:6379', 'localhost:6380', 'localhost:6381', 'localhost:6382'
+  r['urmom'] = 'urmom'
+  r['urdad'] = 'urdad'
+  r['urmom1'] = 'urmom1'
+  r['urdad1'] = 'urdad1'
+  r['urmom2'] = 'urmom2'
+  r['urdad2'] = 'urdad2'
+  r['urmom3'] = 'urmom3'
+  r['urdad3'] = 'urdad3'
+  p r['urmom']
+  p r['urdad']
+  p r['urmom1']
+  p r['urdad1']
+  p r['urmom2']
+  p r['urdad2']
+  p r['urmom3']
+  p r['urdad3']
+  
+  r.push_tail 'listor', 'foo1'
+  r.push_tail 'listor', 'foo2'
+  r.push_tail 'listor', 'foo3'
+  r.push_tail 'listor', 'foo4'
+  r.push_tail 'listor', 'foo5'
+  
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  
+  puts "key distribution:"
+  
+  r.ring.nodes.each do |red|
+    p [red.port, red.keys("*")]
+  end
+  r.delete_cloud!
+  p r.keys('*')
+
+end

data/lib/hash_ring.rb

@@ -0,0 +1,127 @@
+require 'zlib'
+
+class HashRing
+
+  POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+  attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+  # nodes is a list of objects that have a proper to_s representation.
+  # replicas indicates how many virtual points should be used pr. node,
+  # replicas are required to improve the distribution.
+  def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+    @replicas = replicas
+    @ring = {}
+    @nodes = []
+    @sorted_keys = []
+    nodes.each do |node|
+      add_node(node)
+    end
+  end
+  
+  # Adds a `node` to the hash ring (including a number of replicas).
+  def add_node(node)
+    @nodes << node
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{i}")
+      @ring[key] = node
+      @sorted_keys << key
+    end
+    @sorted_keys.sort!
+  end
+  
+  def remove_node(node)
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{count}")
+      @ring.delete(key)
+      @sorted_keys.reject! {|k| k == key}
+    end
+  end
+  
+  # get the node in the hash ring for this key
+  def get_node(key)
+    get_node_pos(key)[0]
+  end
+  
+  def get_node_pos(key)
+    return [nil,nil] if @ring.size == 0
+    crc = Zlib.crc32(key)
+    idx = HashRing.binary_search(@sorted_keys, crc)
+    return [@ring[@sorted_keys[idx]], idx]
+  end
+  
+  def iter_nodes(key)
+    return [nil,nil] if @ring.size == 0
+    node, pos = get_node_pos(key)
+    @sorted_keys[pos..-1].each do |k|
+      yield @ring[k]
+    end  
+  end
+  
+  class << self
+
+    # gem install RubyInline to use this code
+    # Native extension to perform the binary search within the hashring.
+    # There's a pure ruby version below so this is purely optional
+    # for performance.  In testing 20k gets and sets, the native
+    # binary search shaved about 12% off the runtime (9sec -> 8sec).
+    begin
+      require 'inline'
+      inline do |builder|
+        builder.c <<-EOM
+        int binary_search(VALUE ary, unsigned int r) {
+            int upper = RARRAY_LEN(ary) - 1;
+            int lower = 0;
+            int idx = 0;
+
+            while (lower <= upper) {
+                idx = (lower + upper) / 2;
+
+                VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                unsigned int l = NUM2UINT(continuumValue);
+                if (l == r) {
+                    return idx;
+                }
+                else if (l > r) {
+                    upper = idx - 1;
+                }
+                else {
+                    lower = idx + 1;
+                }
+            }
+            return upper;
+        }
+        EOM
+      end
+    rescue Exception => e
+      # Find the closest index in HashRing with value <= the given value
+      def binary_search(ary, value, &block)
+        upper = ary.size - 1
+        lower = 0
+        idx = 0
+
+        while(lower <= upper) do
+          idx = (lower + upper) / 2
+          comp = ary[idx] <=> value
+
+          if comp == 0
+            return idx
+          elsif comp > 0
+            upper = idx - 1
+          else
+            lower = idx + 1
+          end
+        end
+        return upper
+      end
+
+    end
+  end
+
+end
+
+# ring = HashRing.new ['server1', 'server2', 'server3']
+# p ring
+# #
+# p ring.get_node "kjhjkjlkjlkkh"
+# 

data/lib/pipeline.rb

@@ -0,0 +1,31 @@
+require "redis"
+
+class Redis
+  class Pipeline < Redis
+    BUFFER_SIZE = 50_000
+    
+    def initialize(redis)
+      @redis = redis
+      @commands = []
+    end
+    
+    def get_response
+    end
+    
+    def write(data)
+      @commands << data
+      write_and_read if @commands.size >= BUFFER_SIZE
+    end
+    
+    def finish
+      write_and_read
+    end
+    
+    def write_and_read
+      @redis.write @commands.join
+      @redis.read_socket
+      @commands.clear
+    end
+    
+  end
+end

data/lib/redis.rb

@@ -0,0 +1,504 @@
+require 'socket'
+require 'set'
+require File.join(File.dirname(__FILE__),'server')
+require File.join(File.dirname(__FILE__),'pipeline')
+
+
+class RedisError < StandardError
+end
+class RedisRenameError < StandardError
+end
+class Redis
+  ERR = "-".freeze
+  OK = 'OK'.freeze
+  SINGLE = '+'.freeze
+  BULK   = '$'.freeze
+  MULTI  = '*'.freeze
+  INT    = ':'.freeze
+  
+  attr_reader :server
+  
+  
+  def initialize(opts={})
+    @opts = {:host => 'localhost', :port => '6379', :db => 0}.merge(opts)
+    $debug = @opts[:debug]
+    @db = @opts[:db]
+    @server = Server.new(@opts[:host], @opts[:port], (@opts[:timeout]||10))
+  end
+  
+  def pipelined
+    pipeline = Pipeline.new(self)
+    yield pipeline
+    pipeline.finish
+  end
+  
+  def to_s
+    "#{host}:#{port}"
+  end
+  
+  def port
+    @opts[:port]
+  end
+  
+  def host
+    @opts[:host]
+  end
+  
+  def with_socket_management(server, &block)
+    begin
+      block.call(server.socket)
+    #Timeout or server down
+    rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNREFUSED => e
+      server.close
+      puts "Client (#{server.inspect}) disconnected from server: #{e.inspect}\n" if $debug
+      retry
+    #Server down
+    rescue NoMethodError => e
+      puts "Client (#{server.inspect}) tryin server that is down: #{e.inspect}\n Dying!" if $debug
+      raise Errno::ECONNREFUSED
+      #exit
+    end
+  end
+
+  def monitor
+    with_socket_management(@server) do |socket|
+      trap("INT") { puts "\nGot ^C! Dying!"; exit }
+      write "MONITOR\r\n"
+      puts "Now Monitoring..."
+      socket.read(12)
+      loop do
+        x = socket.gets
+        puts x unless x.nil?
+      end
+    end
+  end
+
+  def quit
+    write "QUIT\r\n"
+  end
+  
+  def select_db(index)
+    @db = index
+    write "SELECT #{index}\r\n"
+    get_response
+  end
+  
+  def flush_db
+    write "FLUSHDB\r\n"
+    get_response == OK
+  end    
+
+  def flush_all
+    ensure_retry do
+      puts "Warning!\nFlushing *ALL* databases!\n5 Seconds to Hit ^C!"
+      trap('INT') {quit; return false}
+      sleep 5
+      write "FLUSHALL\r\n"
+      get_response == OK
+    end
+  end
+
+  def last_save
+    write "LASTSAVE\r\n"
+    get_response.to_i
+  end
+  
+  def bgsave
+    write "BGSAVE\r\n"
+    get_response == OK
+  end  
+    
+  def info
+   info = {}
+   write("INFO\r\n")
+   x = get_response
+   x.each do |kv|
+     k,v = kv.split(':', 2)
+     k,v = k.chomp, v = v.chomp
+     info[k.to_sym] = v
+   end
+   info
+  end
+  
+  
+  def bulk_reply
+    begin
+      x = read
+      puts "bulk_reply read value is #{x.inspect}" if $debug
+      return x
+    rescue => e
+      puts "error in bulk_reply #{e}" if $debug
+      nil
+    end
+  end
+  
+  def write(data)
+    with_socket_management(@server) do |socket|
+      puts "writing: #{data}" if $debug
+      socket.write(data)
+    end
+  end
+  
+  def fetch(len)
+    with_socket_management(@server) do |socket|
+      len = [0, len.to_i].max
+      res = socket.read(len + 2)
+      res = res.chomp if res
+      res
+    end
+  end
+  
+  def read(length = read_proto)
+    with_socket_management(@server) do |socket|
+      res = socket.read(length)
+      puts "read is #{res.inspect}" if $debug
+      res
+    end
+  end
+
+  def keys(glob)
+    write "KEYS #{glob}\r\n"
+    get_response.split(' ')
+  end
+
+  def rename!(oldkey, newkey)
+    write "RENAME #{oldkey} #{newkey}\r\n"
+    get_response
+  end  
+  
+  def rename(oldkey, newkey)
+    write "RENAMENX #{oldkey} #{newkey}\r\n"
+    case get_response
+    when -1
+      raise RedisRenameError, "source key: #{oldkey} does not exist"
+    when 0
+      raise RedisRenameError, "target key: #{oldkey} already exists"
+    when -3
+      raise RedisRenameError, "source and destination keys are the same"
+    when 1
+      true
+    end
+  end  
+  
+  def key?(key)
+    write "EXISTS #{key}\r\n"
+    get_response == 1
+  end  
+  
+  def delete(key)
+    write "DEL #{key}\r\n"
+    get_response == 1
+  end  
+  
+  def [](key)
+    get(key)
+  end
+
+  def get(key)
+    write "GET #{key}\r\n"
+    get_response
+  end
+  
+  def mget(*keys)
+    write "MGET #{keys.join(' ')}\r\n"
+    get_response
+  end
+
+  def incr(key, increment=nil)
+    if increment
+      write "INCRBY #{key} #{increment}\r\n"
+    else
+      write "INCR #{key}\r\n"
+    end    
+    get_response
+  end
+
+  def decr(key, decrement=nil)
+    if decrement
+      write "DECRBY #{key} #{decrement}\r\n"
+    else
+      write "DECR #{key}\r\n"
+    end    
+    get_response
+  end
+  
+  def randkey
+    write "RANDOMKEY\r\n"
+    get_response
+  end
+
+  def list_length(key)
+    write "LLEN #{key}\r\n"
+    case i = get_response
+    when -2
+      raise RedisError, "key: #{key} does not hold a list value"
+    else
+      i
+    end
+  end
+
+  def type?(key)
+    write "TYPE #{key}\r\n"
+    get_response
+  end
+  
+  def push_tail(key, string)
+    write "RPUSH #{key} #{string.to_s.size}\r\n#{string.to_s}\r\n"
+    get_response
+  end      
+
+  def push_head(key, string)
+    write "LPUSH #{key} #{string.to_s.size}\r\n#{string.to_s}\r\n"
+    get_response
+  end
+  
+  def pop_head(key)
+    write "LPOP #{key}\r\n"
+    get_response
+  end
+
+  def pop_tail(key)
+    write "RPOP #{key}\r\n"
+    get_response
+  end    
+
+  def list_set(key, index, val)
+    write "LSET #{key} #{index} #{val.to_s.size}\r\n#{val}\r\n"
+    get_response == OK
+  end
+
+  def list_range(key, start, ending)
+    write "LRANGE #{key} #{start} #{ending}\r\n"
+    get_response
+  end
+
+  def list_trim(key, start, ending)
+    write "LTRIM #{key} #{start} #{ending}\r\n"
+    get_response
+  end
+
+  def list_index(key, index)
+    write "LINDEX #{key} #{index}\r\n"
+    get_response
+  end
+
+  def list_rm(key, count, value)
+    write "LREM #{key} #{count} #{value.to_s.size}\r\n#{value}\r\n"
+    case num = get_response
+    when -1
+      raise RedisError, "key: #{key} does not exist"
+    when -2
+      raise RedisError, "key: #{key} does not hold a list value"
+    else
+      num
+    end
+  end 
+
+  def set_add(key, member)
+    write "SADD #{key} #{member.to_s.size}\r\n#{member}\r\n"
+    case get_response
+    when 1
+      true
+    when 0
+      false
+    when -2
+      raise RedisError, "key: #{key} contains a non set value"
+    end
+  end
+
+  def set_delete(key, member)
+    write "SREM #{key} #{member.to_s.size}\r\n#{member}\r\n"
+    case get_response
+    when 1
+      true
+    when 0
+      false
+    when -2
+      raise RedisError, "key: #{key} contains a non set value"
+    end
+  end
+
+  def set_count(key)
+    write "SCARD #{key}\r\n"
+    case i = get_response
+    when -2
+      raise RedisError, "key: #{key} contains a non set value"
+    else
+      i
+    end
+  end
+
+  def set_member?(key, member)
+    write "SISMEMBER #{key} #{member.to_s.size}\r\n#{member}\r\n"
+    case get_response
+    when 1
+      true
+    when 0
+      false
+    when -2
+      raise RedisError, "key: #{key} contains a non set value"
+    end
+  end
+
+  def set_members(key)
+    write "SMEMBERS #{key}\r\n"
+    Set.new(get_response)
+  end
+
+  def set_intersect(*keys)
+    write "SINTER #{keys.join(' ')}\r\n"
+    Set.new(get_response)
+  end
+
+  def set_inter_store(destkey, *keys)
+    write "SINTERSTORE #{destkey} #{keys.join(' ')}\r\n"
+    get_response
+  end
+  
+  def set_union(*keys)
+    write "SUNION #{keys.join(' ')}\r\n"
+    Set.new(get_response)
+  end
+
+  def set_union_store(destkey, *keys)
+    write "SUNIONSTORE #{destkey} #{keys.join(' ')}\r\n"
+    get_response
+  end
+  
+  def set_diff(*keys)
+    write "SDIFF #{keys.join(' ')}\r\n"
+    Set.new(get_response)
+  end
+
+  def set_diff_store(destkey, *keys)
+    write "SDIFFSTORE #{destkey} #{keys.join(' ')}\r\n"
+    get_response
+  end
+
+  def sort(key, opts={})
+    cmd = "SORT #{key}"
+    cmd << " BY #{opts[:by]}" if opts[:by]
+    cmd << " GET #{opts[:get]}" if opts[:get]
+    cmd << " INCR #{opts[:incr]}" if opts[:incr]
+    cmd << " DEL #{opts[:del]}" if opts[:del]
+    cmd << " DECR #{opts[:decr]}" if opts[:decr]
+    cmd << " #{opts[:order]}" if opts[:order]
+    cmd << " LIMIT #{opts[:limit].join(' ')}" if opts[:limit]
+    cmd << "\r\n"
+    write(cmd)
+    get_response
+  end
+      
+  def multi_bulk
+    res = read_proto
+    puts "mb res is #{res.inspect}" if $debug
+    list = []
+    Integer(res).times do
+      vf = get_response
+      puts "curren vf is #{vf.inspect}" if $debug
+      list << vf
+      puts "current list is #{list.inspect}" if $debug
+    end
+    list
+  end
+   
+  def get_reply
+    begin
+      r = read(1)
+      raise RedisError if (r == "\r" || r == "\n")
+    rescue RedisError
+      retry
+    end
+    r
+  end
+   
+  def []=(key, val)
+    set(key,val)
+  end
+  
+
+  def set(key, val, expiry=nil)
+    write("SET #{key} #{val.to_s.size}\r\n#{val}\r\n")
+    s = get_response == OK
+    return expire(key, expiry) if s && expiry
+    s
+  end
+  
+  def expire(key, expiry=nil)
+    write("EXPIRE #{key} #{expiry}\r\n")
+    get_response == 1
+  end
+
+  def set_unless_exists(key, val)
+    write "SETNX #{key} #{val.to_s.size}\r\n#{val}\r\n"
+    get_response == 1
+  end  
+  
+  def status_code_reply
+    begin
+      res = read_proto  
+      if res.index('-') == 0          
+        raise RedisError, res
+      else          
+        true
+      end
+    rescue RedisError
+       raise RedisError
+    end
+  end
+  
+  def get_response
+    begin
+      rtype = get_reply
+    rescue => e
+      raise RedisError, e.inspect
+    end
+    puts "reply_type is #{rtype.inspect}" if $debug
+    case rtype
+    when SINGLE
+      single_line
+    when BULK
+      bulk_reply
+    when MULTI
+      multi_bulk
+    when INT
+      integer_reply
+    when ERR
+      raise RedisError, single_line
+    else
+      raise RedisError, "Unknown response.."
+    end
+  end
+  
+  def integer_reply
+    Integer(read_proto)
+  end
+  
+  def single_line
+    buff = ""
+    while buff[-2..-1] != "\r\n"
+      buff << read(1)
+    end
+    puts "single_line value is #{buff[0..-3].inspect}" if $debug
+    buff[0..-3]
+  end
+  
+  def read_socket
+    with_socket_management(@server) do |socket|
+      while res = socket.read(8096)
+        break if res.size != 8096
+      end
+    end
+  end
+  
+  def read_proto
+    with_socket_management(@server) do |socket|
+      if res = socket.gets
+        x = res.chomp
+        puts "read_proto is #{x.inspect}\n\n" if $debug
+        x.to_i
+      end
+    end
+  end
+  
+end