xunch 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/xunch.rb +25 -0
- data/lib/xunch/cache/cache.rb +88 -0
- data/lib/xunch/cache/cache_builder.rb +68 -0
- data/lib/xunch/cache/field_object_cache.rb +120 -0
- data/lib/xunch/cache/list_field_object_cache.rb +63 -0
- data/lib/xunch/cache/list_object_cache.rb +59 -0
- data/lib/xunch/cache/object_cache.rb +63 -0
- data/lib/xunch/codec/codec.rb +31 -0
- data/lib/xunch/codec/hash_codec.rb +98 -0
- data/lib/xunch/codec/json_codec.rb +81 -0
- data/lib/xunch/shard/redis.rb +270 -0
- data/lib/xunch/shard/shard_info.rb +37 -0
- data/lib/xunch/shard/shard_redis.rb +267 -0
- data/lib/xunch/shard/sharded.rb +50 -0
- data/lib/xunch/utils/exceptions.rb +11 -0
- data/lib/xunch/utils/nginx_cache_helper.rb +52 -0
- data/lib/xunch/utils/rb_tree.rb +634 -0
- data/lib/xunch/utils/rb_tree_node.rb +67 -0
- data/lib/xunch/utils/types.rb +8 -0
- data/lib/xunch/utils/utils.rb +24 -0
- data/test/benchmark_test.rb +68 -0
- data/test/cache_builder_test.rb +28 -0
- data/test/cache_object.rb +120 -0
- data/test/consistency_hash_test.rb +31 -0
- data/test/field_object_cache_test.rb +430 -0
- data/test/hash_codec_test.rb +57 -0
- data/test/json_codec_test.rb +57 -0
- data/test/list_field_object_cache_test.rb +211 -0
- data/test/list_object_cache_test.rb +211 -0
- data/test/nginx_cache_helper_test.rb +45 -0
- data/test/object_cache_test.rb +322 -0
- data/test/rb_tree_test.rb +48 -0
- data/test/redis_benchmark_test.rb +54 -0
- data/test/redis_test.rb +58 -0
- data/test/running_test.rb +212 -0
- data/test/test.rb +176 -0
- data/test/track_record_origin.rb +58 -0
- metadata +125 -0
@@ -0,0 +1,31 @@
|
|
1
|
+
module Xunch
|
2
|
+
class Codec
|
3
|
+
DEFAULT_TYPE_MAP = {
|
4
|
+
"created_at" => :time,
|
5
|
+
"updated_at" => :time,
|
6
|
+
"approved_at" => :time
|
7
|
+
}
|
8
|
+
|
9
|
+
DEFAULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%:z"
|
10
|
+
|
11
|
+
def initialize(klass)
|
12
|
+
raise XunchCodecError.new("Codec class does not defined method '_accessible_attributes', maybe this klass is not a subclass of ActiveRecord::Base.") unless klass.method_defined?(:_accessible_attributes)
|
13
|
+
@klass = klass
|
14
|
+
@type_map = DEFAULT_TYPE_MAP
|
15
|
+
if @klass.const_defined? :TYPE_MAP
|
16
|
+
@type_map = @type_map.merge(@klass::TYPE_MAP)
|
17
|
+
end
|
18
|
+
@set_methods = {}
|
19
|
+
@get_methods = {}
|
20
|
+
@klass.attribute_names.each { |attribute|
|
21
|
+
if attribute.length == 0
|
22
|
+
next
|
23
|
+
end
|
24
|
+
field = Utils.format_field(attribute)
|
25
|
+
@get_methods[field] = attribute.to_sym
|
26
|
+
@set_methods[field] = (attribute + "=").to_sym
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module Xunch
|
2
|
+
class HashCodec < Codec
|
3
|
+
|
4
|
+
def initialize(klass,fields)
|
5
|
+
super(klass)
|
6
|
+
temp_set_methods = {}
|
7
|
+
temp_get_methods = {}
|
8
|
+
@fields = fields
|
9
|
+
@fields.each{ |field|
|
10
|
+
temp_get_method = @get_methods[field]
|
11
|
+
temp_set_method = @set_methods[field]
|
12
|
+
if temp_set_method == nil || temp_get_method == nil
|
13
|
+
raise XunchConfigError.new("field #{field} doesn't have set or get method")
|
14
|
+
end
|
15
|
+
temp_get_methods[field] = temp_get_method
|
16
|
+
temp_set_methods[field] = temp_set_method
|
17
|
+
}
|
18
|
+
@set_methods = temp_set_methods
|
19
|
+
@get_methods = temp_get_methods
|
20
|
+
end
|
21
|
+
|
22
|
+
def encode(value)
|
23
|
+
unless value.class.name == @klass.name
|
24
|
+
raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
|
25
|
+
"object_id is #{@klass.object_id}, but value class is #{value.class}," <<
|
26
|
+
"object_id is #{value.class.object_id}.
|
27
|
+
value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
|
28
|
+
end
|
29
|
+
hash = Hash.new
|
30
|
+
@get_methods.each do | k , v |
|
31
|
+
var_value = value.send(v)
|
32
|
+
if var_value != nil
|
33
|
+
if var_value.class.name == Time.name or var_value.class.name == DateTime.name
|
34
|
+
hash[k] = var_value.strftime(DEFAULT_DATE_FORMAT)
|
35
|
+
else
|
36
|
+
hash[k] = var_value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
return hash
|
41
|
+
end
|
42
|
+
|
43
|
+
def decode(hash)
|
44
|
+
value = @klass.new
|
45
|
+
count = 0
|
46
|
+
hash.each { |k, v|
|
47
|
+
if v == nil
|
48
|
+
count+=1
|
49
|
+
else
|
50
|
+
value.send(@set_methods[k],v)
|
51
|
+
end
|
52
|
+
}
|
53
|
+
if count == hash.length
|
54
|
+
return nil
|
55
|
+
else
|
56
|
+
return value
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def encode_fields(value,fields)
|
61
|
+
hash = Hash.new
|
62
|
+
fields.each { |field|
|
63
|
+
get_method = @get_methods[field]
|
64
|
+
if get_method == nil
|
65
|
+
raise XunchCodecError.new("Undefined field #{field}")
|
66
|
+
end
|
67
|
+
var_value = value.send(get_method)
|
68
|
+
if var_value != nil
|
69
|
+
if var_value.class.name == Time.name or var_value.class.name == DateTime.name
|
70
|
+
hash[field] = var_value.strftime(DEFAULT_DATE_FORMAT)
|
71
|
+
else
|
72
|
+
hash[field] = var_value
|
73
|
+
end
|
74
|
+
end
|
75
|
+
}
|
76
|
+
return hash
|
77
|
+
end
|
78
|
+
|
79
|
+
def decode_fields(hash,fields)
|
80
|
+
value = @klass.new
|
81
|
+
count = 0
|
82
|
+
fields.each { |field|
|
83
|
+
v = hash[field]
|
84
|
+
if v == nil
|
85
|
+
count+=1
|
86
|
+
else
|
87
|
+
value.send(@set_methods[field],v)
|
88
|
+
end
|
89
|
+
}
|
90
|
+
if count == hash.length
|
91
|
+
return nil
|
92
|
+
else
|
93
|
+
return value
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module Xunch
|
2
|
+
class JsonCodec < Codec
|
3
|
+
|
4
|
+
def initialize(klass)
|
5
|
+
super
|
6
|
+
@parser = Yajl::Parser
|
7
|
+
@encoder = Yajl::Encoder
|
8
|
+
end
|
9
|
+
|
10
|
+
def encode(value)
|
11
|
+
unless value.class.name == @klass.name
|
12
|
+
raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
|
13
|
+
"object_id is #{@klass.object_id}, but value class is #{value.class}," <<
|
14
|
+
"object_id is #{value.class.object_id}.
|
15
|
+
value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
|
16
|
+
end
|
17
|
+
hash = Hash.new
|
18
|
+
@get_methods.each do | k , v |
|
19
|
+
var_value = value.send(v)
|
20
|
+
if var_value != nil
|
21
|
+
hash[k] = var_value
|
22
|
+
end
|
23
|
+
end
|
24
|
+
Yajl::Encoder.encode(hash)
|
25
|
+
end
|
26
|
+
|
27
|
+
def decode(json_string)
|
28
|
+
value = @klass.new
|
29
|
+
hash = Yajl::Parser.parse(json_string)
|
30
|
+
hash.each { |k, v|
|
31
|
+
if @type_map.include?(k)
|
32
|
+
case @type_map[k]
|
33
|
+
when :datetime
|
34
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
|
35
|
+
when :time
|
36
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
|
37
|
+
when :bigdecimal
|
38
|
+
v = BigDecimal.new(v.to_s)
|
39
|
+
else
|
40
|
+
# do nothing
|
41
|
+
end
|
42
|
+
end
|
43
|
+
value.send(@set_methods[k],v)
|
44
|
+
}
|
45
|
+
value
|
46
|
+
end
|
47
|
+
|
48
|
+
def encode_fields(value,fields)
|
49
|
+
hash = Hash.new
|
50
|
+
@get_methods.each do | k , v |
|
51
|
+
var_value = value.send(v)
|
52
|
+
if var_value != nil
|
53
|
+
hash[k] = var_value
|
54
|
+
end
|
55
|
+
end
|
56
|
+
hash
|
57
|
+
end
|
58
|
+
|
59
|
+
def decode_fields(hash,fields)
|
60
|
+
value = @klass.new
|
61
|
+
hash = Yajl::Parser.parse(json_string)
|
62
|
+
hash.each { |k, v|
|
63
|
+
if @type_map.include?(k)
|
64
|
+
case @type_map[k]
|
65
|
+
when :datetime
|
66
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
|
67
|
+
when :time
|
68
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
|
69
|
+
when :bigdecimal
|
70
|
+
v = BigDecimal.new(v.to_s)
|
71
|
+
else
|
72
|
+
# do nothing
|
73
|
+
end
|
74
|
+
end
|
75
|
+
value.send(@set_methods[k],v)
|
76
|
+
}
|
77
|
+
value
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
@@ -0,0 +1,270 @@
|
|
1
|
+
#
|
2
|
+
# redis client support redis pool
|
3
|
+
#
|
4
|
+
module Xunch
|
5
|
+
class RedisClient
|
6
|
+
|
7
|
+
DEFAULTS = {
|
8
|
+
:size => 1,
|
9
|
+
:timeout => nil,
|
10
|
+
:pool_timeout => 0,
|
11
|
+
:driver => :hiredis
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
@options = DEFAULTS.merge(options)
|
16
|
+
if RUBY_PLATFORM =~ /mingw/
|
17
|
+
@options[:driver] = nil
|
18
|
+
end
|
19
|
+
if(@options[:pool_timeout] > 0)
|
20
|
+
@pool_timeout = @options[:pool_timeout]
|
21
|
+
else
|
22
|
+
@pool_timeout = 1073741823
|
23
|
+
end
|
24
|
+
@pool = Array.new(@options[:size]) { redis = Redis.new(@options) }
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@resource = ConditionVariable.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def destroy
|
30
|
+
@pool.each {|redis| redis.quit if redis and redis.connected?}
|
31
|
+
end
|
32
|
+
|
33
|
+
def exists(key)
|
34
|
+
with do | redis |
|
35
|
+
redis.exists(key)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def del(*keys)
|
40
|
+
if keys.length > 5
|
41
|
+
with do | redis |
|
42
|
+
redis.pipelined do
|
43
|
+
redis.del(*keys)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
else
|
47
|
+
with do | redis |
|
48
|
+
redis.del(*keys)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def expire(key, ttl)
|
54
|
+
with do | redis |
|
55
|
+
redis.expire(key,ttl)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def ttl(key)
|
60
|
+
with do | redis |
|
61
|
+
redis.ttl(key)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# get value with key
|
66
|
+
#
|
67
|
+
# @param [String] key
|
68
|
+
# @return [String] value cache in redis
|
69
|
+
def get(key)
|
70
|
+
with do | redis |
|
71
|
+
redis.get(key)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def mget(keys)
|
76
|
+
with do | redis |
|
77
|
+
redis.mget(*keys)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# set key value with expire time in second
|
82
|
+
#
|
83
|
+
# @param [String] key
|
84
|
+
# @param [String] value
|
85
|
+
# @param [Fixnum] ttl time to live
|
86
|
+
# @return `"OK"`
|
87
|
+
def set(key, value, ttl)
|
88
|
+
with do | redis |
|
89
|
+
redis.setex(key,ttl,value)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# multi set key value with expire time in second
|
94
|
+
# NOTE: use pipeline inner
|
95
|
+
#
|
96
|
+
# @param [Hash] hash key value pairs
|
97
|
+
# @param [Fixnum] ttl time to live
|
98
|
+
def mset(hash, ttl)
|
99
|
+
with do | redis |
|
100
|
+
if(ttl != 0)
|
101
|
+
redis.pipelined do
|
102
|
+
hash.each { |key,value|
|
103
|
+
redis.setex(key,ttl,value)
|
104
|
+
}
|
105
|
+
end
|
106
|
+
else
|
107
|
+
redis.mset(values)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def hset(key, hash, ttl)
|
113
|
+
with do | redis |
|
114
|
+
redis.pipelined do
|
115
|
+
redis.mapped_hmset(key, hash)
|
116
|
+
if(ttl > 0)
|
117
|
+
redis.expire(key,ttl)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def hget(key, *fields)
|
124
|
+
with do | redis |
|
125
|
+
redis.mapped_hmget(key, *fields)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def hgetall(key)
|
130
|
+
with do | redis |
|
131
|
+
redis.hgetall(key)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def hsetall(key,hash,ttl)
|
136
|
+
with do | redis |
|
137
|
+
redis.pipelined do
|
138
|
+
redis.mapped_hmset(key, hash)
|
139
|
+
if(ttl > 0)
|
140
|
+
redis.expire(key,ttl)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
# multi get hash type keys with fields
|
147
|
+
#
|
148
|
+
# @param keys [Array] redis hash key
|
149
|
+
# @param fields [Array] hash field names
|
150
|
+
def hmget(keys,*fields)
|
151
|
+
# block = lambda { |args| p args }
|
152
|
+
with do | redis |
|
153
|
+
redis.pipelined do
|
154
|
+
keys.each { | key |
|
155
|
+
redis.mapped_hmget(key, *fields)
|
156
|
+
}
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def hmset(hash, ttl)
|
162
|
+
with do | redis |
|
163
|
+
redis.pipelined do
|
164
|
+
hash.each { | key, value |
|
165
|
+
redis.mapped_hmset(key, value)
|
166
|
+
if(ttl > 0)
|
167
|
+
redis.expire(key, ttl)
|
168
|
+
end
|
169
|
+
}
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def hmgetall(keys)
|
175
|
+
with do | redis |
|
176
|
+
redis.pipelined do
|
177
|
+
keys.each { | key |
|
178
|
+
redis.hgetall(key)
|
179
|
+
}
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def llen(key)
|
185
|
+
with do | redis |
|
186
|
+
redis.llen(key)
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def lrem(key,value)
|
191
|
+
with do | redis |
|
192
|
+
redis.lrem(key,1,value)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def lset(temp_key,new_key,values,ttl)
|
197
|
+
with do | redis |
|
198
|
+
redis.pipelined do
|
199
|
+
redis.del(temp_key)
|
200
|
+
values.each {
|
201
|
+
| value |
|
202
|
+
redis.rpush(temp_key,value)
|
203
|
+
}
|
204
|
+
result = redis.rename(temp_key,new_key)
|
205
|
+
if(ttl > 0)
|
206
|
+
redis.expire(new_key,ttl)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def lrange(key, start, stop)
|
213
|
+
with do | redis |
|
214
|
+
redis.lrange(key,start,stop)
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# Rename a key. If the new key already exists it is overwritten.
|
219
|
+
#
|
220
|
+
# @param [String] old_name
|
221
|
+
# @param [String] new_name
|
222
|
+
# @return [String] `OK`
|
223
|
+
def rename(old_key, new_key)
|
224
|
+
with do | redis |
|
225
|
+
redis.rename(old_key,new_key)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def type(key)
|
230
|
+
with do | redis |
|
231
|
+
redis.type(key)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
private
|
236
|
+
def with
|
237
|
+
redis = checkout_redis
|
238
|
+
begin
|
239
|
+
yield redis
|
240
|
+
ensure
|
241
|
+
checkin_redis(redis)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def checkout_redis
|
246
|
+
deadline = Time.now + @pool_timeout
|
247
|
+
@mutex.synchronize do
|
248
|
+
loop do
|
249
|
+
return @pool.pop unless @pool.empty?
|
250
|
+
to_wait = deadline - Time.now
|
251
|
+
raise Timeout::Error, "Waited #{@pool_timeout} seconds" if to_wait <= 0 and @pool_timeout > 0
|
252
|
+
@resource.wait(@mutex, to_wait)
|
253
|
+
end
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def checkin_redis(redis)
|
258
|
+
redis = Redis.new(@options) unless redis and redis.connected?
|
259
|
+
@mutex.synchronize do
|
260
|
+
if @pool.length < @options[:size]
|
261
|
+
@pool.push redis
|
262
|
+
@resource.broadcast
|
263
|
+
else
|
264
|
+
redis.quit
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
end
|
270
|
+
end
|