xunch 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/xunch.rb +25 -0
- data/lib/xunch/cache/cache.rb +88 -0
- data/lib/xunch/cache/cache_builder.rb +68 -0
- data/lib/xunch/cache/field_object_cache.rb +120 -0
- data/lib/xunch/cache/list_field_object_cache.rb +63 -0
- data/lib/xunch/cache/list_object_cache.rb +59 -0
- data/lib/xunch/cache/object_cache.rb +63 -0
- data/lib/xunch/codec/codec.rb +31 -0
- data/lib/xunch/codec/hash_codec.rb +98 -0
- data/lib/xunch/codec/json_codec.rb +81 -0
- data/lib/xunch/shard/redis.rb +270 -0
- data/lib/xunch/shard/shard_info.rb +37 -0
- data/lib/xunch/shard/shard_redis.rb +267 -0
- data/lib/xunch/shard/sharded.rb +50 -0
- data/lib/xunch/utils/exceptions.rb +11 -0
- data/lib/xunch/utils/nginx_cache_helper.rb +52 -0
- data/lib/xunch/utils/rb_tree.rb +634 -0
- data/lib/xunch/utils/rb_tree_node.rb +67 -0
- data/lib/xunch/utils/types.rb +8 -0
- data/lib/xunch/utils/utils.rb +24 -0
- data/test/benchmark_test.rb +68 -0
- data/test/cache_builder_test.rb +28 -0
- data/test/cache_object.rb +120 -0
- data/test/consistency_hash_test.rb +31 -0
- data/test/field_object_cache_test.rb +430 -0
- data/test/hash_codec_test.rb +57 -0
- data/test/json_codec_test.rb +57 -0
- data/test/list_field_object_cache_test.rb +211 -0
- data/test/list_object_cache_test.rb +211 -0
- data/test/nginx_cache_helper_test.rb +45 -0
- data/test/object_cache_test.rb +322 -0
- data/test/rb_tree_test.rb +48 -0
- data/test/redis_benchmark_test.rb +54 -0
- data/test/redis_test.rb +58 -0
- data/test/running_test.rb +212 -0
- data/test/test.rb +176 -0
- data/test/track_record_origin.rb +58 -0
- metadata +125 -0
@@ -0,0 +1,31 @@
|
|
1
|
+
module Xunch
|
2
|
+
class Codec
|
3
|
+
DEFAULT_TYPE_MAP = {
|
4
|
+
"created_at" => :time,
|
5
|
+
"updated_at" => :time,
|
6
|
+
"approved_at" => :time
|
7
|
+
}
|
8
|
+
|
9
|
+
DEFAULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%:z"
|
10
|
+
|
11
|
+
def initialize(klass)
|
12
|
+
raise XunchCodecError.new("Codec class does not defined method '_accessible_attributes', maybe this klass is not a subclass of ActiveRecord::Base.") unless klass.method_defined?(:_accessible_attributes)
|
13
|
+
@klass = klass
|
14
|
+
@type_map = DEFAULT_TYPE_MAP
|
15
|
+
if @klass.const_defined? :TYPE_MAP
|
16
|
+
@type_map = @type_map.merge(@klass::TYPE_MAP)
|
17
|
+
end
|
18
|
+
@set_methods = {}
|
19
|
+
@get_methods = {}
|
20
|
+
@klass.attribute_names.each { |attribute|
|
21
|
+
if attribute.length == 0
|
22
|
+
next
|
23
|
+
end
|
24
|
+
field = Utils.format_field(attribute)
|
25
|
+
@get_methods[field] = attribute.to_sym
|
26
|
+
@set_methods[field] = (attribute + "=").to_sym
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module Xunch
|
2
|
+
class HashCodec < Codec
|
3
|
+
|
4
|
+
def initialize(klass,fields)
|
5
|
+
super(klass)
|
6
|
+
temp_set_methods = {}
|
7
|
+
temp_get_methods = {}
|
8
|
+
@fields = fields
|
9
|
+
@fields.each{ |field|
|
10
|
+
temp_get_method = @get_methods[field]
|
11
|
+
temp_set_method = @set_methods[field]
|
12
|
+
if temp_set_method == nil || temp_get_method == nil
|
13
|
+
raise XunchConfigError.new("field #{field} doesn't have set or get method")
|
14
|
+
end
|
15
|
+
temp_get_methods[field] = temp_get_method
|
16
|
+
temp_set_methods[field] = temp_set_method
|
17
|
+
}
|
18
|
+
@set_methods = temp_set_methods
|
19
|
+
@get_methods = temp_get_methods
|
20
|
+
end
|
21
|
+
|
22
|
+
def encode(value)
|
23
|
+
unless value.class.name == @klass.name
|
24
|
+
raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
|
25
|
+
"object_id is #{@klass.object_id}, but value class is #{value.class}," <<
|
26
|
+
"object_id is #{value.class.object_id}.
|
27
|
+
value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
|
28
|
+
end
|
29
|
+
hash = Hash.new
|
30
|
+
@get_methods.each do | k , v |
|
31
|
+
var_value = value.send(v)
|
32
|
+
if var_value != nil
|
33
|
+
if var_value.class.name == Time.name or var_value.class.name == DateTime.name
|
34
|
+
hash[k] = var_value.strftime(DEFAULT_DATE_FORMAT)
|
35
|
+
else
|
36
|
+
hash[k] = var_value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
return hash
|
41
|
+
end
|
42
|
+
|
43
|
+
def decode(hash)
|
44
|
+
value = @klass.new
|
45
|
+
count = 0
|
46
|
+
hash.each { |k, v|
|
47
|
+
if v == nil
|
48
|
+
count+=1
|
49
|
+
else
|
50
|
+
value.send(@set_methods[k],v)
|
51
|
+
end
|
52
|
+
}
|
53
|
+
if count == hash.length
|
54
|
+
return nil
|
55
|
+
else
|
56
|
+
return value
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def encode_fields(value,fields)
|
61
|
+
hash = Hash.new
|
62
|
+
fields.each { |field|
|
63
|
+
get_method = @get_methods[field]
|
64
|
+
if get_method == nil
|
65
|
+
raise XunchCodecError.new("Undefined field #{field}")
|
66
|
+
end
|
67
|
+
var_value = value.send(get_method)
|
68
|
+
if var_value != nil
|
69
|
+
if var_value.class.name == Time.name or var_value.class.name == DateTime.name
|
70
|
+
hash[field] = var_value.strftime(DEFAULT_DATE_FORMAT)
|
71
|
+
else
|
72
|
+
hash[field] = var_value
|
73
|
+
end
|
74
|
+
end
|
75
|
+
}
|
76
|
+
return hash
|
77
|
+
end
|
78
|
+
|
79
|
+
def decode_fields(hash,fields)
|
80
|
+
value = @klass.new
|
81
|
+
count = 0
|
82
|
+
fields.each { |field|
|
83
|
+
v = hash[field]
|
84
|
+
if v == nil
|
85
|
+
count+=1
|
86
|
+
else
|
87
|
+
value.send(@set_methods[field],v)
|
88
|
+
end
|
89
|
+
}
|
90
|
+
if count == hash.length
|
91
|
+
return nil
|
92
|
+
else
|
93
|
+
return value
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module Xunch
|
2
|
+
class JsonCodec < Codec
|
3
|
+
|
4
|
+
def initialize(klass)
|
5
|
+
super
|
6
|
+
@parser = Yajl::Parser
|
7
|
+
@encoder = Yajl::Encoder
|
8
|
+
end
|
9
|
+
|
10
|
+
def encode(value)
|
11
|
+
unless value.class.name == @klass.name
|
12
|
+
raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
|
13
|
+
"object_id is #{@klass.object_id}, but value class is #{value.class}," <<
|
14
|
+
"object_id is #{value.class.object_id}.
|
15
|
+
value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
|
16
|
+
end
|
17
|
+
hash = Hash.new
|
18
|
+
@get_methods.each do | k , v |
|
19
|
+
var_value = value.send(v)
|
20
|
+
if var_value != nil
|
21
|
+
hash[k] = var_value
|
22
|
+
end
|
23
|
+
end
|
24
|
+
Yajl::Encoder.encode(hash)
|
25
|
+
end
|
26
|
+
|
27
|
+
def decode(json_string)
|
28
|
+
value = @klass.new
|
29
|
+
hash = Yajl::Parser.parse(json_string)
|
30
|
+
hash.each { |k, v|
|
31
|
+
if @type_map.include?(k)
|
32
|
+
case @type_map[k]
|
33
|
+
when :datetime
|
34
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
|
35
|
+
when :time
|
36
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
|
37
|
+
when :bigdecimal
|
38
|
+
v = BigDecimal.new(v.to_s)
|
39
|
+
else
|
40
|
+
# do nothing
|
41
|
+
end
|
42
|
+
end
|
43
|
+
value.send(@set_methods[k],v)
|
44
|
+
}
|
45
|
+
value
|
46
|
+
end
|
47
|
+
|
48
|
+
def encode_fields(value,fields)
|
49
|
+
hash = Hash.new
|
50
|
+
@get_methods.each do | k , v |
|
51
|
+
var_value = value.send(v)
|
52
|
+
if var_value != nil
|
53
|
+
hash[k] = var_value
|
54
|
+
end
|
55
|
+
end
|
56
|
+
hash
|
57
|
+
end
|
58
|
+
|
59
|
+
def decode_fields(hash,fields)
|
60
|
+
value = @klass.new
|
61
|
+
hash = Yajl::Parser.parse(json_string)
|
62
|
+
hash.each { |k, v|
|
63
|
+
if @type_map.include?(k)
|
64
|
+
case @type_map[k]
|
65
|
+
when :datetime
|
66
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
|
67
|
+
when :time
|
68
|
+
v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
|
69
|
+
when :bigdecimal
|
70
|
+
v = BigDecimal.new(v.to_s)
|
71
|
+
else
|
72
|
+
# do nothing
|
73
|
+
end
|
74
|
+
end
|
75
|
+
value.send(@set_methods[k],v)
|
76
|
+
}
|
77
|
+
value
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
@@ -0,0 +1,270 @@
|
|
1
|
+
#
|
2
|
+
# redis client support redis pool
|
3
|
+
#
|
4
|
+
module Xunch
|
5
|
+
class RedisClient
|
6
|
+
|
7
|
+
DEFAULTS = {
|
8
|
+
:size => 1,
|
9
|
+
:timeout => nil,
|
10
|
+
:pool_timeout => 0,
|
11
|
+
:driver => :hiredis
|
12
|
+
}
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
@options = DEFAULTS.merge(options)
|
16
|
+
if RUBY_PLATFORM =~ /mingw/
|
17
|
+
@options[:driver] = nil
|
18
|
+
end
|
19
|
+
if(@options[:pool_timeout] > 0)
|
20
|
+
@pool_timeout = @options[:pool_timeout]
|
21
|
+
else
|
22
|
+
@pool_timeout = 1073741823
|
23
|
+
end
|
24
|
+
@pool = Array.new(@options[:size]) { redis = Redis.new(@options) }
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@resource = ConditionVariable.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def destroy
|
30
|
+
@pool.each {|redis| redis.quit if redis and redis.connected?}
|
31
|
+
end
|
32
|
+
|
33
|
+
def exists(key)
|
34
|
+
with do | redis |
|
35
|
+
redis.exists(key)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def del(*keys)
|
40
|
+
if keys.length > 5
|
41
|
+
with do | redis |
|
42
|
+
redis.pipelined do
|
43
|
+
redis.del(*keys)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
else
|
47
|
+
with do | redis |
|
48
|
+
redis.del(*keys)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def expire(key, ttl)
|
54
|
+
with do | redis |
|
55
|
+
redis.expire(key,ttl)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def ttl(key)
|
60
|
+
with do | redis |
|
61
|
+
redis.ttl(key)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# get value with key
|
66
|
+
#
|
67
|
+
# @param [String] key
|
68
|
+
# @return [String] value cache in redis
|
69
|
+
def get(key)
|
70
|
+
with do | redis |
|
71
|
+
redis.get(key)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def mget(keys)
|
76
|
+
with do | redis |
|
77
|
+
redis.mget(*keys)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# set key value with expire time in second
|
82
|
+
#
|
83
|
+
# @param [String] key
|
84
|
+
# @param [String] value
|
85
|
+
# @param [Fixnum] ttl time to live
|
86
|
+
# @return `"OK"`
|
87
|
+
def set(key, value, ttl)
|
88
|
+
with do | redis |
|
89
|
+
redis.setex(key,ttl,value)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# multi set key value with expire time in second
|
94
|
+
# NOTE: use pipeline inner
|
95
|
+
#
|
96
|
+
# @param [Hash] hash key value pairs
|
97
|
+
# @param [Fixnum] ttl time to live
|
98
|
+
def mset(hash, ttl)
|
99
|
+
with do | redis |
|
100
|
+
if(ttl != 0)
|
101
|
+
redis.pipelined do
|
102
|
+
hash.each { |key,value|
|
103
|
+
redis.setex(key,ttl,value)
|
104
|
+
}
|
105
|
+
end
|
106
|
+
else
|
107
|
+
redis.mset(values)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def hset(key, hash, ttl)
|
113
|
+
with do | redis |
|
114
|
+
redis.pipelined do
|
115
|
+
redis.mapped_hmset(key, hash)
|
116
|
+
if(ttl > 0)
|
117
|
+
redis.expire(key,ttl)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def hget(key, *fields)
|
124
|
+
with do | redis |
|
125
|
+
redis.mapped_hmget(key, *fields)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def hgetall(key)
|
130
|
+
with do | redis |
|
131
|
+
redis.hgetall(key)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def hsetall(key,hash,ttl)
|
136
|
+
with do | redis |
|
137
|
+
redis.pipelined do
|
138
|
+
redis.mapped_hmset(key, hash)
|
139
|
+
if(ttl > 0)
|
140
|
+
redis.expire(key,ttl)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
# multi get hash type keys with fields
|
147
|
+
#
|
148
|
+
# @param keys [Array] redis hash key
|
149
|
+
# @param fields [Array] hash field names
|
150
|
+
def hmget(keys,*fields)
|
151
|
+
# block = lambda { |args| p args }
|
152
|
+
with do | redis |
|
153
|
+
redis.pipelined do
|
154
|
+
keys.each { | key |
|
155
|
+
redis.mapped_hmget(key, *fields)
|
156
|
+
}
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def hmset(hash, ttl)
|
162
|
+
with do | redis |
|
163
|
+
redis.pipelined do
|
164
|
+
hash.each { | key, value |
|
165
|
+
redis.mapped_hmset(key, value)
|
166
|
+
if(ttl > 0)
|
167
|
+
redis.expire(key, ttl)
|
168
|
+
end
|
169
|
+
}
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def hmgetall(keys)
|
175
|
+
with do | redis |
|
176
|
+
redis.pipelined do
|
177
|
+
keys.each { | key |
|
178
|
+
redis.hgetall(key)
|
179
|
+
}
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def llen(key)
|
185
|
+
with do | redis |
|
186
|
+
redis.llen(key)
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def lrem(key,value)
|
191
|
+
with do | redis |
|
192
|
+
redis.lrem(key,1,value)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def lset(temp_key,new_key,values,ttl)
|
197
|
+
with do | redis |
|
198
|
+
redis.pipelined do
|
199
|
+
redis.del(temp_key)
|
200
|
+
values.each {
|
201
|
+
| value |
|
202
|
+
redis.rpush(temp_key,value)
|
203
|
+
}
|
204
|
+
result = redis.rename(temp_key,new_key)
|
205
|
+
if(ttl > 0)
|
206
|
+
redis.expire(new_key,ttl)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def lrange(key, start, stop)
|
213
|
+
with do | redis |
|
214
|
+
redis.lrange(key,start,stop)
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# Rename a key. If the new key already exists it is overwritten.
|
219
|
+
#
|
220
|
+
# @param [String] old_name
|
221
|
+
# @param [String] new_name
|
222
|
+
# @return [String] `OK`
|
223
|
+
def rename(old_key, new_key)
|
224
|
+
with do | redis |
|
225
|
+
redis.rename(old_key,new_key)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def type(key)
|
230
|
+
with do | redis |
|
231
|
+
redis.type(key)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
private
|
236
|
+
def with
|
237
|
+
redis = checkout_redis
|
238
|
+
begin
|
239
|
+
yield redis
|
240
|
+
ensure
|
241
|
+
checkin_redis(redis)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def checkout_redis
|
246
|
+
deadline = Time.now + @pool_timeout
|
247
|
+
@mutex.synchronize do
|
248
|
+
loop do
|
249
|
+
return @pool.pop unless @pool.empty?
|
250
|
+
to_wait = deadline - Time.now
|
251
|
+
raise Timeout::Error, "Waited #{@pool_timeout} seconds" if to_wait <= 0 and @pool_timeout > 0
|
252
|
+
@resource.wait(@mutex, to_wait)
|
253
|
+
end
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def checkin_redis(redis)
|
258
|
+
redis = Redis.new(@options) unless redis and redis.connected?
|
259
|
+
@mutex.synchronize do
|
260
|
+
if @pool.length < @options[:size]
|
261
|
+
@pool.push redis
|
262
|
+
@resource.broadcast
|
263
|
+
else
|
264
|
+
redis.quit
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
end
|
270
|
+
end
|