xunch 0.0.6

data/lib/xunch/shard/shard_info.rb

@@ -0,0 +1,37 @@
+module Xunch
+  class ShardInfo
+    attr_accessor :host, :port, :name, :password, :timeout, :database, :weight
+    
+    DEFAULT_OPTIONS = {
+      :host => "127.0.0.1",
+      :port => 6379,
+      :name => nil,
+      :password => nil,
+      :timeout => nil,
+      :db => 0,
+      :weight => 1
+    }
+
+    def initialize(options)
+      raise "initialize ShardInfo error, options can not be nil." if options == nil
+      @options = DEFAULT_OPTIONS.merge(options)
+
+      @host = options[:host]
+      @port = options[:port]
+      @name = options[:name]
+      @password = options[:password]
+      @timeout = options[:timeout]
+      @db = options[:db]
+      @weight = options[:weight]
+    end
+
+    def create_resource
+      RedisClient.new(@options)
+    end
+
+    def to_s
+      "#{@host}:#{@port}::#{@db}*#{@weight}";
+    end
+
+  end
+end

data/lib/xunch/shard/shard_redis.rb

@@ -0,0 +1,267 @@
+module Xunch
+  class ShardRedis < Sharded
+
+    def initialize(regex,shard_infos)
+      super 
+    end
+
+    def exists(key)
+      redis = get_shard(key)
+      redis.exists(key)
+    end
+
+    def del(key)
+      redis = get_shard(key)
+      redis.del(key)
+    end
+
+    def batch_del(keys)
+      shard_key_map = Hash.new
+      shard_index_map = Hash.new
+      for i in 0 .. keys.length - 1 do
+        redis = get_shard(keys[i])
+        if shard_key_map[redis] == nil
+          shard_key = Array.new
+          shard_key.push(keys[i])
+          shard_index = Array.new
+          shard_index.push(i)
+          shard_key_map[redis] = shard_key
+          shard_index_map[redis] = shard_index
+        else
+          shard_key_map[redis].push(keys[i])
+          shard_index_map[redis].push(i)
+        end
+      end
+
+      results = Array.new(keys.length)
+      shard_key_map.each { | redis, keys |
+        values = redis.del(keys)
+        shard_index = shard_index_map[redis]
+        for i in 0 .. shard_index.length - 1 do
+          results[shard_index[i]] = values[i]
+        end
+      }
+      results
+    end
+
+    def expire(key, ttl)
+      redis = get_shard(key)
+      redis.pexpire(key, ttl)
+    end
+    
+    def get_expire(key)
+      redis = get_shard(key)
+      redis.pttl(key)
+    end
+
+    def get(key)
+      redis = get_shard(key)
+      redis.get(key)
+    end
+
+    def mget(keys)
+      shard_key_map = Hash.new
+      shard_index_map = Hash.new
+      for i in 0 .. keys.length - 1 do
+        redis = get_shard(keys[i])
+        if shard_key_map[redis] == nil
+          shard_key = Array.new
+          shard_key.push(keys[i])
+          shard_index = Array.new
+          shard_index.push(i)
+          shard_key_map[redis] = shard_key
+          shard_index_map[redis] = shard_index
+        else
+          shard_key_map[redis].push(keys[i])
+          shard_index_map[redis].push(i)
+        end
+      end
+
+      results = Array.new(keys.length)
+      shard_key_map.each { | redis, keys |
+        values = redis.mget(keys)
+        shard_index = shard_index_map[redis]
+        for i in 0 .. shard_index.length - 1 do
+          results[shard_index[i]] = values[i]
+        end
+      }
+      results
+    end
+
+    def set(key, value, ttl)
+      redis = get_shard(key)
+      redis.set(key,value,ttl)
+    end
+
+    def mset(key_value_pairs, ttl)
+      shard_map = Hash.new
+      key_value_pairs.each { | key, value |
+        redis = get_shard(key)
+        if shard_map[redis] == nil
+          shard = Hash.new
+          shard[key] = value
+          shard_map[redis] = shard
+        else
+          shard_map[redis][key] = value
+        end
+      }
+      result = []
+      shard_map.each { | redis, kvs |
+        result.push redis.mset(kvs, ttl)
+      }
+      return result.flatten!
+    end
+
+    def mapped_hget(key,fields)
+      redis = get_shard(key)
+      redis.hget(key,*fields)
+    end
+
+    def mapped_hmget(keys,fields)
+      shard_key_map = Hash.new
+      shard_index_map = Hash.new
+      for i in 0 .. keys.length - 1 do
+        redis = get_shard(keys[i])
+        if shard_key_map[redis] == nil
+          shard_key = Array.new
+          shard_key.push(keys[i])
+          shard_index = Array.new
+          shard_index.push(i)
+          shard_key_map[redis] = shard_key
+          shard_index_map[redis] = shard_index
+        else
+          shard_key_map[redis].push(keys[i])
+          shard_index_map[redis].push(i)
+        end
+      end
+
+      results = Array.new(keys.length)
+      shard_key_map.each { | redis, keys |
+        values = redis.hmget(keys,*fields)
+        shard_index = shard_index_map[redis]
+        for i in 0 .. shard_index.length - 1 do
+          results[shard_index[i]] = values[i]
+        end
+      }
+      results
+    end
+
+    def hgetall(key)
+      redis = get_shard(key)
+      redis.hgetall(key)
+    end
+
+    def hsetall(key, hash, ttl)
+      redis = get_shard(key)
+      redis.hsetall(key,hash,ttl)
+    end
+
+    def hmgetall(keys)
+      shard_key_map = Hash.new
+      shard_index_map = Hash.new
+      for i in 0 .. keys.length - 1 do
+        redis = get_shard(keys[i])
+        if shard_key_map[redis] == nil
+          shard_key = Array.new
+          shard_key.push(keys[i])
+          shard_index = Array.new
+          shard_index.push(i)
+          shard_key_map[redis] = shard_key
+          shard_index_map[redis] = shard_index
+        else
+          shard_key_map[redis].push(keys[i])
+          shard_index_map[redis].push(i)
+        end
+      end
+
+      results = Array.new(keys.length)
+      shard_key_map.each { | redis, keys |
+        values = redis.hmgetall(keys)
+        shard_index = shard_index_map[redis]
+        for i in 0 .. shard_index.length - 1 do
+          results[shard_index[i]] = values[i]
+        end
+      }
+      results
+    end
+
+    def hmsetall(key_value_pairs, ttl)
+      shard_map = Hash.new
+      key_value_pairs.each { | key, value |
+        redis = get_shard(key)
+        if shard_map[redis] == nil
+          shard = Hash.new
+          shard[key] = value
+          shard_map[redis] = shard
+        else
+          shard_map[redis][key] = value
+        end
+      }
+      result = []
+      shard_map.each { | redis, kvs |
+        result.push redis.hmset(kvs, ttl)
+      }
+      result.flatten!
+    end
+
+    def mapped_hmget(keys,fields)
+      shard_key_map = Hash.new
+      shard_index_map = Hash.new
+      for i in 0 .. keys.length - 1 do
+        redis = get_shard(keys[i])
+        if shard_key_map[redis] == nil
+          shard_key = Array.new
+          shard_key.push(keys[i])
+          shard_index = Array.new
+          shard_index.push(i)
+          shard_key_map[redis] = shard_key
+          shard_index_map[redis] = shard_index
+        else
+          shard_key_map[redis].push(keys[i])
+          shard_index_map[redis].push(i)
+        end
+      end
+
+      results = Array.new(keys.length)
+      shard_key_map.each { | redis, keys |
+        values = redis.hmget(keys,*fields)
+        shard_index = shard_index_map[redis]
+        for i in 0 .. shard_index.length - 1 do
+          results[shard_index[i]] = values[i]
+        end
+      }
+      results
+    end
+
+    def llen(key)
+      redis = get_shard(key)
+      redis.llen(key)
+    end
+
+    def lremove(key,value)
+      redis = get_shard(key)
+      redis.lrem(key)
+    end
+
+    def lset(temp_key, new_key, sub_keys, ttl)
+      redis = get_shard(new_key)
+      redis.lset(temp_key,new_key,sub_keys,ttl)
+    end
+
+    def lrange(key, start, stop)
+      redis = get_shard(key)
+      redis.lrange(key,start,stop)
+    end
+
+    def rename(old_key, new_key)
+      redis = get_shard(key)
+      redis.rename(old_key,new_key)
+    end
+
+    def type(key)
+      redis = get_shard(key)
+      redis.type(key)
+    end
+
+  end
+end

data/lib/xunch/shard/sharded.rb

@@ -0,0 +1,50 @@
+module Xunch
+  # maintain a consistency hash ring 
+  # this ring must be same as other language xunch sharded
+  class Sharded
+    include Murmurhash
+
+    def initialize(regex,shard_infos)
+      @regexp = Regexp.new(regex)
+      @nodes = RBTree.new
+      @resources = Hash.new
+      for i in 0 .. shard_infos.length - 1 do
+        if (shard_infos[i].name == nil)
+          for n in 0 .. 160 * shard_infos[i].weight do
+            @nodes.put(Murmurhash.hash2A("SHARD-#{i}-NODE-#{n}"), shard_infos[i]);
+          end
+        else
+          for n in 0 .. 160 * shard_infos[i].weight do
+            @nodes.put(Murmurhash.hash2A("#{shard_infos[i].name}-NODE-#{n}"), shard_infos[i]);
+          end
+        end
+        @resources.store(shard_infos[i], shard_infos[i].create_resource);
+      end
+    end
+
+    def destroy
+      @resources.each { |shard_info, redis_client|
+        redis_client.destroy
+      }
+      @nodes.clear
+      @resources.clear
+    end
+
+    protected
+    def get_shard(key)
+      shard_info = @nodes.ceiling_value(Murmurhash.hash2A(get_keytag(key)))
+      if(shard_info == nil)
+        shard_info = @nodes.first_value
+      end
+      shard_redis = @resources[shard_info]
+      return shard_redis
+    end
+
+    def get_keytag(key)
+      if @regexp != nil
+        key = @regexp.match(key)[0]
+      end
+      return key
+    end
+  end
+end

data/lib/xunch/utils/exceptions.rb

@@ -0,0 +1,11 @@
+module Xunch
+  class XunchError < StandardError
+  end
+
+  class XunchCodecError < XunchError
+  end
+
+  class XunchConfigError < XunchError
+  end
+
+end

data/lib/xunch/utils/nginx_cache_helper.rb

@@ -0,0 +1,52 @@
+require 'net/http'
+
+module Xunch
+  class NginxCacheHelper
+
+    def initialize(domain,port = nil,timeout = nil)
+      if domain == nil || domain.strip.empty?
+        raise ArgumentError.new("domain can't be nil or empty string.")
+      end
+      if port == nil
+        port = 80
+      else
+        port = port.to_i
+      end
+      @http = Net::HTTP.new(domain, port)
+      if timeout != nil
+        @http.read_timeout = timeout.to_i
+      else
+        @http.read_timeout = 30
+      end
+    end
+
+    def evict(path)
+      respose = nil
+      begin
+        respose = @http.get("/purge" + path)
+      rescue SocketError => e
+        raise e
+      end
+      if respose == nil || respose.code != "200"
+        return false
+      else
+        return true
+      end
+    end
+
+    def cache(path)
+      respose = nil
+      begin
+        respose = @http.get(path)
+      rescue SocketError => e
+        raise e
+      end
+      if respose == nil || respose.code != "200"
+        return false
+      else
+        return true
+      end
+    end
+  
+  end
+end

data/lib/xunch/utils/rb_tree.rb

@@ -0,0 +1,634 @@
+module Xunch
+  class RBTree
+
+
+    # The tree's root node.
+    attr_reader :root
+    
+    # The number of nodes in the tree.
+    attr_reader :size
+
+    # The key compare proc
+    attr_reader :compare_proc
+
+    # Creates a new tree.
+    def initialize(&compare_proc)
+      @size = 0
+      @root = nil
+      @compare_proc = compare_proc
+    end
+
+    def put(key,value)
+      tmp_node = @root
+      if(tmp_node == nil)
+        compare_key(key,key)
+        @root = RBTree::Node.new(key,value,nil)
+        @size = 1
+        return value
+      end
+      cmp = 0
+      parent = nil
+
+      begin
+        parent = tmp_node
+        cmp = compare_key(key,parent.key)
+        if cmp < 0
+          tmp_node = tmp_node.left;
+        elsif cmp > 0
+          tmp_node = tmp_node.right;
+        else
+          return tmp_node.value = value;
+        end
+      end while tmp_node != nil
+
+      node = RBTree::Node.new(key,value,parent)
+
+      if cmp < 0
+        parent.left = node
+      else
+        parent.right = node
+      end
+      
+      fixafterinsertion(node)    
+      @size+=1
+
+      return value
+      
+    end
+
+    def remove(key)
+      p = getnode(key)
+      if (p == nil)
+        return nil
+      end
+
+      oldValue = p.value
+      deletenode(p)
+      return oldValue
+    end
+
+    def get(key)
+      p = getnode(key);
+      return (p == nil ? nil : p.value)
+    end
+
+    def clear
+      @size = 0
+      @root = nil
+    end
+
+    def size
+      @size
+    end
+
+    def first_node
+      p = @root;
+      if p != nil
+        while p.left != nil
+          p = p.left
+        end
+      end
+      RBTree::ImmutableNode.new(p.key,p.value)
+    end
+
+    def first_key
+      p = @root;
+      if p != nil
+        while p.left != nil
+          p = p.left
+        end
+      end
+      p.key
+    end
+
+    def first_value
+      p = @root;
+      if p != nil
+        while p.left != nil
+          p = p.left
+        end
+      end
+      p.value
+    end
+
+    def last_node
+      p = @root;
+      if p != nil
+        while p.right != nil
+          p = p.right
+        end
+      end
+      RBTree::ImmutableNode.new(p.key,p.value)
+    end
+
+    def last_key
+      p = @root;
+      if p != nil
+        while p.right != nil
+          p = p.right
+        end
+      end
+      p.key
+    end
+
+    def last_value
+      p = @root;
+      if p != nil
+        while p.right != nil
+          p = p.right
+        end
+      end
+      p.value
+    end
+
+    def floor_node(key)
+      p = @root;
+      while p != nil
+        cmp = compare_key(key, p.key)
+        if cmp > 0
+          if p.right != nil
+            p = p.right
+          else
+            if p == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(p.key,p.value)
+            end
+          end
+        elsif cmp < 0
+          if p.left != nil
+            p = p.left
+          else
+            parent = p.parent
+            ch = p
+            while parent != nil && ch == parent.left
+              ch = parent
+              parent = parent.parent;
+            end
+            if parent == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(parent.key,parent.value)
+            end
+          end
+        else
+          if p == nil
+            return nil
+          else
+            return RBTree::ImmutableNode.new(p.key,p.value)
+          end
+        end
+      end
+    
+      return nil
+    end
+
+    def floor_key(key)
+      node = floor_node(key)
+      if node == nil
+        return nil
+      else
+        return node.key
+      end
+    end
+
+    def floor_value(key)
+      node = floor_node(key)
+      if node == nil
+        return nil
+      else
+        return node.value
+      end
+    end
+
+    def lower_node(key)
+      p = @root;
+      while p != nil
+        cmp = compare_key(key, p.key)
+        if cmp > 0
+          if p.right != nil
+            p = p.right
+          else
+            if p == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(p.key,p.value)
+            end
+          end
+        else
+          if p.left != nil
+            p = p.left
+          else
+            parent = p.parent
+            ch = p
+            while parent != nil && ch == parent.left
+              ch = parent
+              parent = parent.parent;
+            end
+            if parent == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(parent.key,parent.value)
+            end
+          end
+        end
+      end
+      return nil
+    end
+
+    def lower_key(key)
+      node = lower_node(key)
+      if node == nil
+        return nil
+      else
+        return node.key
+      end
+    end
+
+    def lower_value(key)
+      node = lower_node(key)
+      if node == nil
+        return nil
+      else
+        return node.value
+      end
+    end
+
+    def ceiling_node(key)
+      p = @root;
+      while p != nil
+        cmp = compare_key(key, p.key)
+        if cmp < 0
+          if p.left != nil
+            p = p.left
+          else
+            if p == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(p.key,p.value)
+            end
+          end
+        elsif cmp > 0
+          if p.right != nil
+            p = p.right
+          else
+            parent = p.parent
+            ch = p
+            while parent != nil && ch == parent.right
+              ch = parent
+              parent = parent.parent;
+            end
+            if parent == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(parent.key,parent.value)
+            end
+          end
+        else
+          if p == nil
+            return nil
+          else
+            return RBTree::ImmutableNode.new(p.key,p.value)
+          end
+        end
+      end
+    
+      return nil
+    end
+
+    def ceiling_key(key)
+      node = ceiling_node(key)
+      if node == nil
+        return nil
+      else
+        return node.key
+      end
+    end
+
+    def ceiling_value(key)
+      node = ceiling_node(key)
+      if node == nil
+        return nil
+      else
+        return node.value
+      end
+    end
+
+    def higher_node(key)
+      p = @root;
+      while p != nil
+        cmp = compare_key(key, p.key)
+        if cmp < 0
+          if p.left != nil
+            p = p.left
+          else
+            if p == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(p.key,p.value)
+            end
+          end
+        else
+          if p.right != nil
+            p = p.right
+          else
+            parent = p.parent
+            ch = p
+            while parent != nil && ch == parent.right
+              ch = parent
+              parent = parent.parent;
+            end
+            if parent == nil
+              return nil
+            else
+              return RBTree::ImmutableNode.new(parent.key,parent.value)
+            end
+          end
+        end
+      end
+    
+      return nil
+    end
+
+    def higher_key(key)
+      node = higher_node(key)
+      if node == nil
+        return nil
+      else
+        return node.key
+      end
+    end
+
+    def higher_value(key)
+      node = higher_node(key)
+      if node == nil
+        return nil
+      else
+        return node.value
+      end
+    end
+
+    def successor(t)
+      if t == nil
+        return nil
+      elsif t.right != nil
+        p = t.right
+        while p.left != nil
+            p = p.left
+        end
+        return p
+      else
+        p = t.parent
+        ch = t
+        while p != nil && ch == p.right
+          ch = p
+          p = p.parent
+        end
+        return p
+      end
+    end
+
+    def deletenode(p)
+      @size-=1
+
+      if p.left != nil && p.right != nil
+        s = successor(p)
+        p.key = s.key
+        p.value = s.value
+        p = s
+      end
+
+      replacement = (p.left != nil ? p.left : p.right)
+
+      if replacement != nil
+        replacement.parent = p.parent;
+        if p.parent == nil
+          @root = replacement
+        elsif p == p.parent.left
+          p.parent.left  = replacement
+        else
+          p.parent.right = replacement
+        end
+
+        p.left = p.right = p.parent = nil
+
+        if p.color == :black
+          fixafterdeletion(replacement)
+        end
+      elsif p.parent == nil
+        @root = nil
+      else
+        if p.color == :black
+          fixdfterdeletion(p)
+        end
+        if p.parent != nil
+          if p == p.parent.left
+            p.parent.left = nil
+          elsif p == p.parent.right
+            p.parent.right = nil
+          end
+          p.parent = nil
+        end
+      end
+    end
+
+    def getnode(key)
+      raise "key is nil." unless key != nil 
+      p = @root;
+      while p != nil
+        cmp = compare_key(key, p.key)
+        if cmp < 0
+          p = p.left
+        elsif cmp > 0
+          p = p.right
+        else
+          return p
+        end
+      end
+      return nil
+    end
+
+    def fixafterdeletion(x)
+      while x != @root && colorof(x) == :black
+        if x == leftof(parentof(x))
+          sib = rightof(parentof(x))
+
+          if colorof(sib) == :red
+            setcolor(sib, :black)
+            setcolor(parentof(x), :red)
+            rotateleft(parentof(x))
+            sib = rightof(parentof(x))
+          end
+
+          if colorof(leftof(sib))  == :black && colorof(rightof(sib)) == :black
+            setcolor(sib, :red)
+            x = parentof(x)
+          else
+            if colorof(rightof(sib)) == :black
+              setcolor(leftof(sib), :black)
+              setcolor(sib, :red)
+              rotateright(sib)
+              sib = rightof(parentof(x))
+            end
+            setcolor(sib, colorof(parentof(x)))
+            setcolor(parentof(x), :black)
+            setcolor(rightof(sib), :black)
+            rotateleft(parentof(x))
+            x = @root
+          end
+        else
+          sib = leftof(parentof(x))
+
+          if colorof(sib) == :red
+            setcolor(sib, :black)
+            setcolor(parentof(x), :red)
+            rotateright(parentof(x))
+            sib = leftof(parentof(x))
+          end
+
+          if colorof(rightof(sib)) == :black && colorof(leftof(sib)) == :black
+            setcolor(sib, :red)
+            x = parentof(x)
+          else
+            if colorof(leftof(sib)) == :black
+              setcolor(rightof(sib), :black)
+              setcolor(sib, :red)
+              rotateleft(sib)
+              sib = leftof(parentof(x))
+            end
+            setcolor(sib, colorof(parentof(x)))
+            setcolor(parentof(x), :black)
+            setcolor(leftof(sib), :black)
+            rotateright(parentof(x))
+            x = @root;
+          end
+        end
+      end
+      setcolor(x, :black);
+    end
+
+    def fixafterinsertion(x)
+      x.color = :red
+
+      while x != nil && x != @root && x.parent.color == :red 
+        if parentof(x) == leftof(parentof(parentof(x)))
+          y = rightof(parentof(parentof(x)))
+          if colorof(y) == :red
+            setcolor(parentof(x), :black)
+            setcolor(y, :black)
+            setcolor(parentof(parentof(x)), :red)
+            x = parentof(parentof(x))
+          else
+            if x == rightof(parentof(x)) 
+              x = parentof(x)
+              rotateleft(x)
+            end
+            setcolor(parentof(x), :black);
+            setcolor(parentof(parentof(x)), :red);
+            rotateright(parentof(parentof(x)));
+          end
+        else
+          y = leftof(parentof(parentof(x)))
+          if colorof(y) == :red
+            setcolor(parentof(x), :black)
+            setcolor(y, :black)
+            setcolor(parentof(parentof(x)), :red)
+            x = parentof(parentof(x))
+          else
+            if x == leftof(parentof(x))
+              x = parentof(x)
+              rotateright(x)
+            end
+            setcolor(parentof(x), :black);
+            setcolor(parentof(parentof(x)), :red);
+            rotateleft(parentof(parentof(x)));
+          end
+        end
+      end
+
+      @root.color = :black;
+    end
+
+    def rotateleft(p)
+      if p != nil
+        r = p.right
+        p.right = r.left
+        if r.left != nil
+          r.left.parent = p
+        end
+        r.parent = p.parent
+        if p.parent == nil
+          @root = r
+        elsif p.parent.left == p
+          p.parent.left = r
+        else
+          p.parent.right = r
+        end
+        r.left = p
+        p.parent = r
+      end
+    end
+
+    def rotateright(p)
+      if p != nil
+        l = p.left
+        p.left = l.right
+        if l.right != nil
+          l.right.parent = p
+        end
+        l.parent = p.parent
+        if p.parent == nil
+          @root = l
+        elsif p.parent.right == p
+          p.parent.right = l
+        else 
+          p.parent.left = l
+        end
+        l.right = p
+        p.parent = l
+      end
+    end
+
+    def parentof(node)
+      return (node == nil ? nil : node.parent)
+    end
+
+    def leftof(node)
+      return (node == nil ? nil : node.left)
+    end
+
+    def rightof(node)
+      return (node == nil ? nil : node.right)
+    end
+
+    def colorof(node)
+      if node != nil
+        node.color
+      end
+    end
+
+    def setcolor(node, color)
+      if node != nil
+        node.color = color
+      end 
+    end
+
+    def compare_key(key1,key2)
+      if !key1.integer? || !key2.integer?
+        raise "#{key1} type is #{key1.class}, #{key2} type is #{key2.class}, but need type Integer"
+      end
+      if @compare_proc != nil
+        return @cmp_proc.call(key1, key2)
+      else
+        if key1 > key2
+          return 1
+        elsif key1 < key2
+          return -1
+        elsif key1 == key2
+          return 0
+        end
+      end
+    end
+
+  end
+end