xunch 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 11095eadd1ac6cf93128811cd1bab2d818e345b1
+  data.tar.gz: 8e60835af4879fd33dfa4fe7cfc67d009548d5c2
+SHA512:
+  metadata.gz: 6b7ca3c02e48930d5d7b551a8393c677493d3ac74ba6dc69fbf54249233ff2804d62f353e93324b62449b255b76635e39d25a8a4ba98dca17ee3a697e9d934fa
+  data.tar.gz: fed73c21df8039c4622c9ca91328f66aa5cc933f61a2230af0bd5c870804b41a1e502abeee45f875652168bcb04158947c2a2c1a6f734e183351d69f8e1041a3

data/lib/xunch.rb

@@ -0,0 +1,25 @@
+require 'yajl'
+require 'date'
+require 'bigdecimal'
+require 'murmurhash'
+require 'redis'
+require 'yaml'
+require 'xunch/utils/rb_tree'
+require 'xunch/utils/rb_tree_node'
+require 'xunch/utils/exceptions'
+require 'xunch/utils/types'
+require 'xunch/utils/utils'
+require 'xunch/utils/nginx_cache_helper'
+require 'xunch/shard/shard_info'
+require 'xunch/shard/sharded'
+require 'xunch/shard/shard_redis'
+require 'xunch/shard/redis'
+require 'xunch/codec/codec'
+require 'xunch/codec/json_codec'
+require 'xunch/codec/hash_codec'
+require 'xunch/cache/cache'
+require 'xunch/cache/object_cache'
+require 'xunch/cache/field_object_cache'
+require 'xunch/cache/list_field_object_cache'
+require 'xunch/cache/list_object_cache'
+require 'xunch/cache/cache_builder'

data/lib/xunch/cache/cache.rb

@@ -0,0 +1,88 @@
+module Xunch
+  class Cache
+
+    def initialize(options, shard_infos)
+      @options = initialize_options(options)
+      @shard_redis = ShardRedis.new(@options[:regex],shard_infos)
+    end
+
+    def evict(key)
+      @shard_redis.del(assembleKey(key))
+    end
+
+    def batch_evict(keys)
+      new_keys = []
+      keys.each { |key|
+        new_keys.push assembleKey(key)
+      }
+      @shard_redis.batch_del(new_keys)
+    end
+
+    def destroy
+      @shard_redis.destroy
+    end
+
+    protected 
+    def getCacheObjectKey(value)
+      key = value.send(@options[:key_field_name])
+      raise ArgumentError("value #{value} key field can not be nil.") unless key != nil
+      key
+    end
+
+    def assembleKey(key)
+      name = @options[:name]
+      name + "_".concat(key.to_s).concat("_").concat(@options[:version].to_s)
+    end
+
+    def assembleTempKey(key)
+      name = @options[:name]
+      name + "_".concat(key.to_s).concat("_").concat(@options[:version].to_s).concat("_temp")
+    end
+
+    def initialize_options(options)
+      use_options = {}
+      expire_time = 60 * 60 * 24 * 1000
+      if(options["expire_time"] != nil)
+        expire_time = options["expire_time"]
+      end
+      use_options.store(:expire_time, expire_time)
+
+      if(options["cache_class"] != nil)
+        cache_class = eval(options["cache_class"])
+      elsif options["type"] != CacheType::LISTOBJECT && options["type"] != CacheType::LISTFIELDOBJECT
+        raise ArgumentError, "cache_class is nil"
+      end
+      use_options.store(:cache_class, cache_class)
+
+      version = 1
+      if(options["version"] != nil)
+        version = options["version"]
+        use_options.store(:version, version) 
+      end
+
+      if(options["name"] != nil)
+        name = options["name"]
+        use_options.store(:name, name)
+      end
+
+      if(options["key_field_name"] != nil)
+        key_field_name = options["key_field_name"]
+        use_options.store(:key_field_name, key_field_name)
+      end
+
+      if(options["timeout"] != nil)
+        timeout = options["timeout"]
+        use_options.store(:timeout, timeout)
+      end
+
+      regex = '[0-9]+'
+      if(options["regex"] != nil)
+        regex = options["regex"]
+      end
+      use_options.store(:regex, regex)
+
+      return use_options
+    end
+
+  end
+end

data/lib/xunch/cache/cache_builder.rb

@@ -0,0 +1,68 @@
+module Xunch
+  class CacheBuilder
+    # every cache config hava attributes below:
+    # ['type'] => the cache type, have object|field_object|list_object
+    # ['name'] => the cache name, used for redis key prefix
+    # ['version'] => the cache version, maybe you change your cache object format and you will change a version of the cache
+    # ['expire_time'] => cache expire time for every key in millisecond
+    # ['cache_class'] => which type of object you cache, this will help us to encode/decode your object
+    # ['key_field_name'] => define which field in the object we used for redis key
+    # ['regex'] => a regex string which used for match consistency hash key
+    # ['shard_infos'] => define which redis instance we used for caching
+    #
+    #
+    def self.build(file)
+      configs = YAML.load_file(file)
+      shard_info_configs = configs["shard_infos"]
+      cache_configs = configs["caches"]
+
+      shards = {}
+      shard_info_configs.each_value  { |shard_info_config|
+        options = {}
+        shard_info_config.each { |k,v|
+            options[k.to_sym] = v
+        }
+        shards[options[:name]] = ShardInfo.new(options)
+      }
+      caches = {}
+      lazy_caches = {}
+      cache_configs.each_value  { |cache_config|
+        shard_names = cache_config["shards"].split(",")
+        shard_infos = []
+        shard_names.each { |shard_name|
+          shard_infos.push(shards[shard_name])
+        }
+        case cache_config["type"]
+        when CacheType::OBJECT
+          cache = Xunch::ObjectCache.new(cache_config,shard_infos)
+          caches[cache_config["name"]] = cache
+        when CacheType::FIELDOBJECT
+          cache = Xunch::FieldObjectCache.new(cache_config,shard_infos)
+          caches[cache_config["name"]] = cache
+        when CacheType::LISTOBJECT
+          lazy_caches[cache_config] = shard_infos
+        when CacheType::LISTFIELDOBJECT
+          lazy_caches[cache_config] = shard_infos
+        else
+          raise XunchConfigError.new("Unknown cache type #{cache_config["type"]}.")
+        end
+      }
+      lazy_caches.each { |cache_config,shard_infos|
+        delegate = caches[cache_config["delegate"]]
+        raise XunchConfigError.new("list_cache init error, delegate does not exist.") unless delegate != nil
+        cache = nil
+        case cache_config["type"]
+        when CacheType::LISTOBJECT
+          cache = Xunch::ListObjectCache.new(cache_config,shard_infos,delegate)
+        when CacheType::LISTFIELDOBJECT
+          cache = Xunch::ListFieldObjectCache.new(cache_config,shard_infos,delegate)
+        else
+          raise XunchConfigError.new("Unknown cache type #{cache_config["type"]}.")
+        end
+        
+        caches[cache_config["name"]] = cache
+      }
+      caches
+    end
+  end
+end

data/lib/xunch/cache/field_object_cache.rb

@@ -0,0 +1,120 @@
+module Xunch
+  #
+  #
+  #
+  #
+  class FieldObjectCache < Cache
+
+    def initialize(options, shard_infos)
+      super
+      fields = options['fields']
+      if fields == nil
+        raise XunchConfigError.new("fields can not be nil")
+      end
+      fields_array = fields.split(",")
+      if fields_array.length == 1 and fields_array[0].strip.empty?
+        raise XunchConfigError.new("fields config error")
+      end
+      formatted_fields_array = Utils.format_fields(fields_array)
+      @fields_array = formatted_fields_array
+      @codec = HashCodec.new(@options[:cache_class],formatted_fields_array)
+    end
+
+    def get(key)
+      redis_key = assembleKey(key)
+      data = @shard_redis.mapped_hget(redis_key,@fields_array)
+      @codec.decode(data)
+    end
+
+    def put(value)
+      putex(value,@options[:expire_time])
+    end
+
+    def putex(value, ttl)
+      if(value == nil)
+        return
+      end
+      key = getCacheObjectKey(value)
+      key = assembleKey(key)
+      data = @codec.encode(value)
+      @shard_redis.hsetall(key,data,ttl)
+    end
+
+    def multi_get(keys)
+      redis_keys = Array.new(keys.length)
+      for i in 0 .. keys.length - 1 do
+        redis_keys[i] = assembleKey(keys[i])
+      end
+      datas = @shard_redis.mapped_hmget(redis_keys,@fields_array)
+      for i in 0 .. datas.length - 1 do
+        datas[i] = @codec.decode(datas[i])
+      end
+      return datas
+    end
+
+    def multi_put(values)
+      multi_putex(values,@options[:expire_time])
+    end
+
+    def multi_putex(values,ttl)
+      kvs = Hash.new
+      values.each { | value |
+        if value == nil
+          next
+        end
+        key = getCacheObjectKey(value)
+        redis_key = assembleKey(key);
+        kvs[redis_key] = @codec.encode(value);
+      }
+      @shard_redis.hmsetall(kvs, ttl);
+    end
+
+    def get_with_field(key, fields)
+      redis_key = assembleKey(key)
+      data = @shard_redis.mapped_hget(redis_key,fields)
+      @codec.decode_fields(data,fields)
+    end
+
+    def put_with_field(value, fields)
+      put_with_field_ex(value,@options[:expire_time],fields)
+    end
+
+    def put_with_field_ex(value, ttl, fields)
+      if(value == nil)
+        return
+      end
+      key = getCacheObjectKey(value)
+      key = assembleKey(key)
+      data = @codec.encode_fields(value,fields)
+      @shard_redis.hsetall(key,data,ttl)
+    end
+
+    def multi_put_with_field_ex(values, ttl, fields)
+      kvs = Hash.new
+      values.each { | value |
+        if value == nil
+          next
+        end
+        key = getCacheObjectKey(value)
+        key = assembleKey(key);
+        kvs[key] = @codec.encode_fields(value,fields);
+      }
+      @shard_redis.hmsetall(kvs, ttl);
+    end
+
+    def multi_put_with_field(values, fields)
+      multi_put_with_field_ex(values,@options[:expire_time],fields)
+    end
+
+    def multi_get_with_field(keys, fields)
+      for i in 0 .. keys.length - 1 do
+        keys[i] = assembleKey(keys[i])
+      end
+      datas = @shard_redis.mapped_hmget(keys,fields)
+      for i in 0 .. datas.length - 1 do
+        datas[i] = @codec.decode_fields(datas[i],fields)
+      end
+      return datas
+    end
+  end
+end

data/lib/xunch/cache/list_field_object_cache.rb

@@ -0,0 +1,63 @@
+module Xunch
+  #列表缓存目前不支持并发写入，未来也不打算支持并发
+  #主要的使用场景是发现页热门的单线程写入和并发的读取
+  #并且提供remove接口，帮助从列表中移除已经不存在的声音，用户，专辑
+  class ListFieldObjectCache < Cache
+
+    def initialize(options, shard_infos, object_cache)
+      super(options,shard_infos)
+      @delegate = object_cache
+    end
+
+    # 查询接口
+    # @key 列表的key
+    # @page 页码
+    # @size 页大小
+    #
+    def get(key, page, size)
+      raise "key can not be nil." unless key != nil
+      raise "page must be a positive number." unless page > 0
+      raise "size must be a positive number and less than 100." unless page != nil or size < 100
+      start = (page - 1) * size;
+      stop = page * size - 1;
+      new_key = assembleKey(key)
+      object_keys = @shard_redis.lrange(new_key,start,stop)
+      hash = {}
+      hash["keys"] = object_keys
+      hash["values"] = @delegate.multi_get(object_keys)
+      hash
+    end
+
+    def put(key, values)
+      raise "key can not be nil." unless key != nil
+      raise "values can not be nil." unless values != nil
+      sub_keys = []
+      values.each { | value |
+        raise "value in values can not be nil." unless value != nil
+        sub_keys.push(@delegate.getCacheObjectKey(value))
+      }
+      temp_key = assembleTempKey(key)
+      new_key = assembleKey(key)
+      @delegate.multi_putex(values,@options[:expire_time])
+      @shard_redis.lset(temp_key,new_key,sub_keys,@options[:expire_time])
+    end
+
+    def remove(key, sub_key)
+      raise "key can not be nil." unless key != nil
+      raise "sub_key can not be nil." unless sub_key != nil
+      new_key = assembleKey(key)
+      @shard_redis.lremove(new_key,sub_key)
+    end
+
+    def size(key)
+      raise "key can not be nil." unless key != nil
+      new_key = assembleKey(key)
+      @shard_redis.llen(new_key)
+    end
+
+    def delegate
+      @delegate
+    end
+
+  end
+end

data/lib/xunch/cache/list_object_cache.rb

@@ -0,0 +1,59 @@
+module Xunch
+  #列表缓存目前不支持并发写入，未来也不打算支持并发
+  #主要的使用场景是发现页热门的单线程写入和并发的读取
+  #并且提供remove接口，帮助从列表中移除已经不存在的声音，用户，专辑
+  class ListObjectCache < Cache
+
+    def initialize(options, shard_infos, object_cache)
+      super(options,shard_infos)
+      @delegate = object_cache
+    end
+
+    # 查询接口
+    # @key 列表的key
+    # @page 页码
+    # @size 页大小
+    #
+    def get(key, page, size)
+      raise "key can not be nil." unless key != nil
+      raise "page must be a positive number." unless page > 0
+      raise "size must be a positive number and less than 100." unless page != nil or size < 100
+      start = (page - 1) * size;
+      stop = page * size - 1;
+      new_key = assembleKey(key)
+      object_keys = @shard_redis.lrange(new_key,start,stop)
+      hash = {}
+      hash["keys"] = object_keys
+      hash["values"] = @delegate.multi_get(object_keys)
+      hash
+    end
+
+    def put(key, values)
+      raise "key can not be nil." unless key != nil
+      raise "values can not be nil." unless values != nil
+      sub_keys = []
+      values.each { | value |
+        raise "value in values can not be nil." unless value != nil
+        sub_keys.push(@delegate.getCacheObjectKey(value))
+      }
+      temp_key = assembleTempKey(key)
+      new_key = assembleKey(key)
+      @delegate.multi_putex(values,@options[:expire_time])
+      @shard_redis.lset(temp_key,new_key,sub_keys,@options[:expire_time])
+    end
+
+    def remove(key, sub_key)
+      raise "key can not be nil." unless key != nil
+      raise "sub_key can not be nil." unless sub_key != nil
+      new_key = assembleKey(key)
+      @shard_redis.lremove(new_key,sub_key)
+    end
+
+    def size(key)
+      raise "key can not be nil." unless key != nil
+      new_key = assembleKey(key)
+      @shard_redis.llen(new_key)
+    end
+
+  end
+end

data/lib/xunch/cache/object_cache.rb

@@ -0,0 +1,63 @@
+module Xunch
+  class ObjectCache < Cache
+
+    def initialize(options, shard_infos)
+      super
+      @codec = JsonCodec.new(@options[:cache_class])
+    end
+
+    def get(key)
+      redis_key = assembleKey(key)
+      data = @shard_redis.get(redis_key)
+      if data != nil
+        @codec.decode(data)
+      end
+    end
+
+    def put(value)
+      putex(value,@options[:expire_time])
+    end
+
+    def putex(value, ttl)
+      if(value == nil)
+        return
+      end
+      key = getCacheObjectKey(value)
+      key = assembleKey(key)
+      data = @codec.encode(value)
+      @shard_redis.set(key,data,ttl.to_s)
+    end
+
+    def multi_get(keys)
+      redis_keys = Array.new(keys.length)
+      for i in 0 .. keys.length - 1 do
+        redis_keys[i] = assembleKey(keys[i])
+      end
+      datas = @shard_redis.mget(redis_keys)
+      for i in 0 .. datas.length - 1 do
+        if datas[i] != nil
+          datas[i] = @codec.decode(datas[i])
+        end
+      end
+      datas
+    end
+
+    def multi_put(values)
+      multi_putex(values,@options[:expire_time])
+    end
+
+    def multi_putex(values,ttl)
+      kvs = Hash.new
+      values.each { | value |
+        if value == nil
+          next
+        end
+        key = getCacheObjectKey(value)
+        key = assembleKey(key);
+        kvs[key] = @codec.encode(value);
+      }
+      @shard_redis.mset(kvs, ttl);
+    end
+
+  end
+end