xunch 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/lib/xunch.rb +25 -0
  3. data/lib/xunch/cache/cache.rb +88 -0
  4. data/lib/xunch/cache/cache_builder.rb +68 -0
  5. data/lib/xunch/cache/field_object_cache.rb +120 -0
  6. data/lib/xunch/cache/list_field_object_cache.rb +63 -0
  7. data/lib/xunch/cache/list_object_cache.rb +59 -0
  8. data/lib/xunch/cache/object_cache.rb +63 -0
  9. data/lib/xunch/codec/codec.rb +31 -0
  10. data/lib/xunch/codec/hash_codec.rb +98 -0
  11. data/lib/xunch/codec/json_codec.rb +81 -0
  12. data/lib/xunch/shard/redis.rb +270 -0
  13. data/lib/xunch/shard/shard_info.rb +37 -0
  14. data/lib/xunch/shard/shard_redis.rb +267 -0
  15. data/lib/xunch/shard/sharded.rb +50 -0
  16. data/lib/xunch/utils/exceptions.rb +11 -0
  17. data/lib/xunch/utils/nginx_cache_helper.rb +52 -0
  18. data/lib/xunch/utils/rb_tree.rb +634 -0
  19. data/lib/xunch/utils/rb_tree_node.rb +67 -0
  20. data/lib/xunch/utils/types.rb +8 -0
  21. data/lib/xunch/utils/utils.rb +24 -0
  22. data/test/benchmark_test.rb +68 -0
  23. data/test/cache_builder_test.rb +28 -0
  24. data/test/cache_object.rb +120 -0
  25. data/test/consistency_hash_test.rb +31 -0
  26. data/test/field_object_cache_test.rb +430 -0
  27. data/test/hash_codec_test.rb +57 -0
  28. data/test/json_codec_test.rb +57 -0
  29. data/test/list_field_object_cache_test.rb +211 -0
  30. data/test/list_object_cache_test.rb +211 -0
  31. data/test/nginx_cache_helper_test.rb +45 -0
  32. data/test/object_cache_test.rb +322 -0
  33. data/test/rb_tree_test.rb +48 -0
  34. data/test/redis_benchmark_test.rb +54 -0
  35. data/test/redis_test.rb +58 -0
  36. data/test/running_test.rb +212 -0
  37. data/test/test.rb +176 -0
  38. data/test/track_record_origin.rb +58 -0
  39. metadata +125 -0
@@ -0,0 +1,31 @@
1
+ module Xunch
2
+ class Codec
3
+ DEFAULT_TYPE_MAP = {
4
+ "created_at" => :time,
5
+ "updated_at" => :time,
6
+ "approved_at" => :time
7
+ }
8
+
9
+ DEFAULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%:z"
10
+
11
+ def initialize(klass)
12
+ raise XunchCodecError.new("Codec class does not defined method '_accessible_attributes', maybe this klass is not a subclass of ActiveRecord::Base.") unless klass.method_defined?(:_accessible_attributes)
13
+ @klass = klass
14
+ @type_map = DEFAULT_TYPE_MAP
15
+ if @klass.const_defined? :TYPE_MAP
16
+ @type_map = @type_map.merge(@klass::TYPE_MAP)
17
+ end
18
+ @set_methods = {}
19
+ @get_methods = {}
20
+ @klass.attribute_names.each { |attribute|
21
+ if attribute.length == 0
22
+ next
23
+ end
24
+ field = Utils.format_field(attribute)
25
+ @get_methods[field] = attribute.to_sym
26
+ @set_methods[field] = (attribute + "=").to_sym
27
+ }
28
+ end
29
+
30
+ end
31
+ end
@@ -0,0 +1,98 @@
1
+ module Xunch
2
+ class HashCodec < Codec
3
+
4
+ def initialize(klass,fields)
5
+ super(klass)
6
+ temp_set_methods = {}
7
+ temp_get_methods = {}
8
+ @fields = fields
9
+ @fields.each{ |field|
10
+ temp_get_method = @get_methods[field]
11
+ temp_set_method = @set_methods[field]
12
+ if temp_set_method == nil || temp_get_method == nil
13
+ raise XunchConfigError.new("field #{field} doesn't have set or get method")
14
+ end
15
+ temp_get_methods[field] = temp_get_method
16
+ temp_set_methods[field] = temp_set_method
17
+ }
18
+ @set_methods = temp_set_methods
19
+ @get_methods = temp_get_methods
20
+ end
21
+
22
+ def encode(value)
23
+ unless value.class.name == @klass.name
24
+ raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
25
+ "object_id is #{@klass.object_id}, but value class is #{value.class}," <<
26
+ "object_id is #{value.class.object_id}.
27
+ value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
28
+ end
29
+ hash = Hash.new
30
+ @get_methods.each do | k , v |
31
+ var_value = value.send(v)
32
+ if var_value != nil
33
+ if var_value.class.name == Time.name or var_value.class.name == DateTime.name
34
+ hash[k] = var_value.strftime(DEFAULT_DATE_FORMAT)
35
+ else
36
+ hash[k] = var_value
37
+ end
38
+ end
39
+ end
40
+ return hash
41
+ end
42
+
43
+ def decode(hash)
44
+ value = @klass.new
45
+ count = 0
46
+ hash.each { |k, v|
47
+ if v == nil
48
+ count+=1
49
+ else
50
+ value.send(@set_methods[k],v)
51
+ end
52
+ }
53
+ if count == hash.length
54
+ return nil
55
+ else
56
+ return value
57
+ end
58
+ end
59
+
60
+ def encode_fields(value,fields)
61
+ hash = Hash.new
62
+ fields.each { |field|
63
+ get_method = @get_methods[field]
64
+ if get_method == nil
65
+ raise XunchCodecError.new("Undefined field #{field}")
66
+ end
67
+ var_value = value.send(get_method)
68
+ if var_value != nil
69
+ if var_value.class.name == Time.name or var_value.class.name == DateTime.name
70
+ hash[field] = var_value.strftime(DEFAULT_DATE_FORMAT)
71
+ else
72
+ hash[field] = var_value
73
+ end
74
+ end
75
+ }
76
+ return hash
77
+ end
78
+
79
+ def decode_fields(hash,fields)
80
+ value = @klass.new
81
+ count = 0
82
+ fields.each { |field|
83
+ v = hash[field]
84
+ if v == nil
85
+ count+=1
86
+ else
87
+ value.send(@set_methods[field],v)
88
+ end
89
+ }
90
+ if count == hash.length
91
+ return nil
92
+ else
93
+ return value
94
+ end
95
+ end
96
+ end
97
+ end
98
+
@@ -0,0 +1,81 @@
1
+ module Xunch
2
+ class JsonCodec < Codec
3
+
4
+ def initialize(klass)
5
+ super
6
+ @parser = Yajl::Parser
7
+ @encoder = Yajl::Encoder
8
+ end
9
+
10
+ def encode(value)
11
+ unless value.class.name == @klass.name
12
+ raise XunchCodecError.new("Codec error. Codec class is #{@klass}," <<
13
+ "object_id is #{@klass.object_id}, but value class is #{value.class}," <<
14
+ "object_id is #{value.class.object_id}.
15
+ value class = #{value.class.inspect} vs config class = #{@klass.inspect} ")
16
+ end
17
+ hash = Hash.new
18
+ @get_methods.each do | k , v |
19
+ var_value = value.send(v)
20
+ if var_value != nil
21
+ hash[k] = var_value
22
+ end
23
+ end
24
+ Yajl::Encoder.encode(hash)
25
+ end
26
+
27
+ def decode(json_string)
28
+ value = @klass.new
29
+ hash = Yajl::Parser.parse(json_string)
30
+ hash.each { |k, v|
31
+ if @type_map.include?(k)
32
+ case @type_map[k]
33
+ when :datetime
34
+ v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
35
+ when :time
36
+ v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
37
+ when :bigdecimal
38
+ v = BigDecimal.new(v.to_s)
39
+ else
40
+ # do nothing
41
+ end
42
+ end
43
+ value.send(@set_methods[k],v)
44
+ }
45
+ value
46
+ end
47
+
48
+ def encode_fields(value,fields)
49
+ hash = Hash.new
50
+ @get_methods.each do | k , v |
51
+ var_value = value.send(v)
52
+ if var_value != nil
53
+ hash[k] = var_value
54
+ end
55
+ end
56
+ hash
57
+ end
58
+
59
+ def decode_fields(hash,fields)
60
+ value = @klass.new
61
+ hash = Yajl::Parser.parse(json_string)
62
+ hash.each { |k, v|
63
+ if @type_map.include?(k)
64
+ case @type_map[k]
65
+ when :datetime
66
+ v = DateTime.strptime(v, DEFAULT_DATE_FORMAT)
67
+ when :time
68
+ v = DateTime.strptime(v, DEFAULT_DATE_FORMAT).to_time
69
+ when :bigdecimal
70
+ v = BigDecimal.new(v.to_s)
71
+ else
72
+ # do nothing
73
+ end
74
+ end
75
+ value.send(@set_methods[k],v)
76
+ }
77
+ value
78
+ end
79
+ end
80
+ end
81
+
@@ -0,0 +1,270 @@
1
+ #
2
+ # redis client support redis pool
3
+ #
4
+ module Xunch
5
+ class RedisClient
6
+
7
+ DEFAULTS = {
8
+ :size => 1,
9
+ :timeout => nil,
10
+ :pool_timeout => 0,
11
+ :driver => :hiredis
12
+ }
13
+
14
+ def initialize(options = {})
15
+ @options = DEFAULTS.merge(options)
16
+ if RUBY_PLATFORM =~ /mingw/
17
+ @options[:driver] = nil
18
+ end
19
+ if(@options[:pool_timeout] > 0)
20
+ @pool_timeout = @options[:pool_timeout]
21
+ else
22
+ @pool_timeout = 1073741823
23
+ end
24
+ @pool = Array.new(@options[:size]) { redis = Redis.new(@options) }
25
+ @mutex = Mutex.new
26
+ @resource = ConditionVariable.new
27
+ end
28
+
29
+ def destroy
30
+ @pool.each {|redis| redis.quit if redis and redis.connected?}
31
+ end
32
+
33
+ def exists(key)
34
+ with do | redis |
35
+ redis.exists(key)
36
+ end
37
+ end
38
+
39
+ def del(*keys)
40
+ if keys.length > 5
41
+ with do | redis |
42
+ redis.pipelined do
43
+ redis.del(*keys)
44
+ end
45
+ end
46
+ else
47
+ with do | redis |
48
+ redis.del(*keys)
49
+ end
50
+ end
51
+ end
52
+
53
+ def expire(key, ttl)
54
+ with do | redis |
55
+ redis.expire(key,ttl)
56
+ end
57
+ end
58
+
59
+ def ttl(key)
60
+ with do | redis |
61
+ redis.ttl(key)
62
+ end
63
+ end
64
+
65
+ # get value with key
66
+ #
67
+ # @param [String] key
68
+ # @return [String] value cache in redis
69
+ def get(key)
70
+ with do | redis |
71
+ redis.get(key)
72
+ end
73
+ end
74
+
75
+ def mget(keys)
76
+ with do | redis |
77
+ redis.mget(*keys)
78
+ end
79
+ end
80
+
81
+ # set key value with expire time in second
82
+ #
83
+ # @param [String] key
84
+ # @param [String] value
85
+ # @param [Fixnum] ttl time to live
86
+ # @return `"OK"`
87
+ def set(key, value, ttl)
88
+ with do | redis |
89
+ redis.setex(key,ttl,value)
90
+ end
91
+ end
92
+
93
+ # multi set key value with expire time in second
94
+ # NOTE: use pipeline inner
95
+ #
96
+ # @param [Hash] hash key value pairs
97
+ # @param [Fixnum] ttl time to live
98
+ def mset(hash, ttl)
99
+ with do | redis |
100
+ if(ttl != 0)
101
+ redis.pipelined do
102
+ hash.each { |key,value|
103
+ redis.setex(key,ttl,value)
104
+ }
105
+ end
106
+ else
107
+ redis.mset(values)
108
+ end
109
+ end
110
+ end
111
+
112
+ def hset(key, hash, ttl)
113
+ with do | redis |
114
+ redis.pipelined do
115
+ redis.mapped_hmset(key, hash)
116
+ if(ttl > 0)
117
+ redis.expire(key,ttl)
118
+ end
119
+ end
120
+ end
121
+ end
122
+
123
+ def hget(key, *fields)
124
+ with do | redis |
125
+ redis.mapped_hmget(key, *fields)
126
+ end
127
+ end
128
+
129
+ def hgetall(key)
130
+ with do | redis |
131
+ redis.hgetall(key)
132
+ end
133
+ end
134
+
135
+ def hsetall(key,hash,ttl)
136
+ with do | redis |
137
+ redis.pipelined do
138
+ redis.mapped_hmset(key, hash)
139
+ if(ttl > 0)
140
+ redis.expire(key,ttl)
141
+ end
142
+ end
143
+ end
144
+ end
145
+
146
+ # multi get hash type keys with fields
147
+ #
148
+ # @param keys [Array] redis hash key
149
+ # @param fields [Array] hash field names
150
+ def hmget(keys,*fields)
151
+ # block = lambda { |args| p args }
152
+ with do | redis |
153
+ redis.pipelined do
154
+ keys.each { | key |
155
+ redis.mapped_hmget(key, *fields)
156
+ }
157
+ end
158
+ end
159
+ end
160
+
161
+ def hmset(hash, ttl)
162
+ with do | redis |
163
+ redis.pipelined do
164
+ hash.each { | key, value |
165
+ redis.mapped_hmset(key, value)
166
+ if(ttl > 0)
167
+ redis.expire(key, ttl)
168
+ end
169
+ }
170
+ end
171
+ end
172
+ end
173
+
174
+ def hmgetall(keys)
175
+ with do | redis |
176
+ redis.pipelined do
177
+ keys.each { | key |
178
+ redis.hgetall(key)
179
+ }
180
+ end
181
+ end
182
+ end
183
+
184
+ def llen(key)
185
+ with do | redis |
186
+ redis.llen(key)
187
+ end
188
+ end
189
+
190
+ def lrem(key,value)
191
+ with do | redis |
192
+ redis.lrem(key,1,value)
193
+ end
194
+ end
195
+
196
+ def lset(temp_key,new_key,values,ttl)
197
+ with do | redis |
198
+ redis.pipelined do
199
+ redis.del(temp_key)
200
+ values.each {
201
+ | value |
202
+ redis.rpush(temp_key,value)
203
+ }
204
+ result = redis.rename(temp_key,new_key)
205
+ if(ttl > 0)
206
+ redis.expire(new_key,ttl)
207
+ end
208
+ end
209
+ end
210
+ end
211
+
212
+ def lrange(key, start, stop)
213
+ with do | redis |
214
+ redis.lrange(key,start,stop)
215
+ end
216
+ end
217
+
218
+ # Rename a key. If the new key already exists it is overwritten.
219
+ #
220
+ # @param [String] old_name
221
+ # @param [String] new_name
222
+ # @return [String] `OK`
223
+ def rename(old_key, new_key)
224
+ with do | redis |
225
+ redis.rename(old_key,new_key)
226
+ end
227
+ end
228
+
229
+ def type(key)
230
+ with do | redis |
231
+ redis.type(key)
232
+ end
233
+ end
234
+
235
+ private
236
+ def with
237
+ redis = checkout_redis
238
+ begin
239
+ yield redis
240
+ ensure
241
+ checkin_redis(redis)
242
+ end
243
+ end
244
+
245
+ def checkout_redis
246
+ deadline = Time.now + @pool_timeout
247
+ @mutex.synchronize do
248
+ loop do
249
+ return @pool.pop unless @pool.empty?
250
+ to_wait = deadline - Time.now
251
+ raise Timeout::Error, "Waited #{@pool_timeout} seconds" if to_wait <= 0 and @pool_timeout > 0
252
+ @resource.wait(@mutex, to_wait)
253
+ end
254
+ end
255
+ end
256
+
257
+ def checkin_redis(redis)
258
+ redis = Redis.new(@options) unless redis and redis.connected?
259
+ @mutex.synchronize do
260
+ if @pool.length < @options[:size]
261
+ @pool.push redis
262
+ @resource.broadcast
263
+ else
264
+ redis.quit
265
+ end
266
+ end
267
+ end
268
+
269
+ end
270
+ end