xunch 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/lib/xunch.rb +25 -0
  3. data/lib/xunch/cache/cache.rb +88 -0
  4. data/lib/xunch/cache/cache_builder.rb +68 -0
  5. data/lib/xunch/cache/field_object_cache.rb +120 -0
  6. data/lib/xunch/cache/list_field_object_cache.rb +63 -0
  7. data/lib/xunch/cache/list_object_cache.rb +59 -0
  8. data/lib/xunch/cache/object_cache.rb +63 -0
  9. data/lib/xunch/codec/codec.rb +31 -0
  10. data/lib/xunch/codec/hash_codec.rb +98 -0
  11. data/lib/xunch/codec/json_codec.rb +81 -0
  12. data/lib/xunch/shard/redis.rb +270 -0
  13. data/lib/xunch/shard/shard_info.rb +37 -0
  14. data/lib/xunch/shard/shard_redis.rb +267 -0
  15. data/lib/xunch/shard/sharded.rb +50 -0
  16. data/lib/xunch/utils/exceptions.rb +11 -0
  17. data/lib/xunch/utils/nginx_cache_helper.rb +52 -0
  18. data/lib/xunch/utils/rb_tree.rb +634 -0
  19. data/lib/xunch/utils/rb_tree_node.rb +67 -0
  20. data/lib/xunch/utils/types.rb +8 -0
  21. data/lib/xunch/utils/utils.rb +24 -0
  22. data/test/benchmark_test.rb +68 -0
  23. data/test/cache_builder_test.rb +28 -0
  24. data/test/cache_object.rb +120 -0
  25. data/test/consistency_hash_test.rb +31 -0
  26. data/test/field_object_cache_test.rb +430 -0
  27. data/test/hash_codec_test.rb +57 -0
  28. data/test/json_codec_test.rb +57 -0
  29. data/test/list_field_object_cache_test.rb +211 -0
  30. data/test/list_object_cache_test.rb +211 -0
  31. data/test/nginx_cache_helper_test.rb +45 -0
  32. data/test/object_cache_test.rb +322 -0
  33. data/test/rb_tree_test.rb +48 -0
  34. data/test/redis_benchmark_test.rb +54 -0
  35. data/test/redis_test.rb +58 -0
  36. data/test/running_test.rb +212 -0
  37. data/test/test.rb +176 -0
  38. data/test/track_record_origin.rb +58 -0
  39. metadata +125 -0
@@ -0,0 +1,58 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+
3
+ require "test/unit"
4
+ require "xunch"
5
+
6
+ class RedisTest < Test::Unit::TestCase
7
+ include Test::Unit::Assertions
8
+ def setup
9
+ super
10
+ options = {
11
+ :host => "192.168.1.174",
12
+ :port => 6379,
13
+ :timeout => 5.0,
14
+ :password => 'jredis123456',
15
+ :db => 10,
16
+ :pool_timeout => 12,
17
+ :size => 10
18
+ }
19
+ @redis_client = Xunch::RedisClient.new(options)
20
+ puts "setup"
21
+ end
22
+
23
+ def test_redis
24
+ assert_equal('OK',@redis_client.set('key1','value1',100000))
25
+ assert(@redis_client.exists('key1'),"expect exists key1")
26
+ assert((@redis_client.ttl('key1')<100000),"expect ttl less than 100000")
27
+ assert(@redis_client.expire('key1',1000000))
28
+ assert(@redis_client.ttl('key1')>100000,"expect ttl more than 100000")
29
+ assert(@redis_client.ttl('key1')<1000000,"expect ttl less than 1000000")
30
+ assert_equal('value1',@redis_client.get('key1'))
31
+ assert(@redis_client.del('key1'),"del key1 didn't success")
32
+
33
+ hash = {"k1"=>"v1","k2"=>"v2","k3"=>"v3","k4"=>"v4","k5"=>"v5"}
34
+ assert_equal(["OK", true, true, true, true, true],@redis_client.mset(hash,1000000))
35
+
36
+ assert_equal(["v1", "v2", "v3", "v4", "v5", nil],@redis_client.mget("k1","k2","k3","k4","k5","k6"))
37
+
38
+ fields = {"field1"=>"v1","field2"=>"v2","field3"=>"v3","field4"=>"v4","field5"=>"v5"}
39
+ fieldkeys = ["field1","field2","field3","field4","field5"]
40
+
41
+ assert_equal(["OK", true], @redis_client.hset("hash_key1",fields,10000))
42
+ assert_equal({"field1"=>"v1", "field2"=>"v2", "field3"=>"v3", "field4"=>"v4", "field5"=>"v5"}, @redis_client.hget("hash_key1",*fieldkeys))
43
+ assert_equal({"field1"=>"v1", "field2"=>"v2", "field3"=>"v3", "field4"=>"v4", "field5"=>"v5"}, @redis_client.hgetall("hash_key1"))
44
+
45
+ hash = {"hash_k1"=>fields,"hash_k2"=>fields,"hash_k3"=>fields,"hash_k4"=>fields,"hash_k5"=>fields}
46
+ hashkeys =["hash_k1","hash_k2","hash_k0","hash_k8","hash_k5"]
47
+ assert_equal(["OK", true, "OK", true, "OK", true, "OK", true, "OK", true], @redis_client.hmset(hash,100000))
48
+
49
+ @redis_client.hmget(hashkeys,*fieldkeys)
50
+
51
+ end
52
+
53
+ def teardown
54
+ super
55
+ @redis_client.destroy
56
+ puts "teardown"
57
+ end
58
+ end
@@ -0,0 +1,212 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+ $:.unshift File.expand_path("../../test", __FILE__)
3
+ require "test/unit"
4
+ require "xunch"
5
+ require 'yaml'
6
+ require 'cache_object'
7
+ require 'bigdecimal'
8
+ require 'hessian2'
9
+ require 'track_record_origin'
10
+
11
+ class RunningCacheTest < Test::Unit::TestCase
12
+ include Test::Unit::Assertions
13
+ def setup
14
+ super
15
+ @fields = ["createdAt","updatedAt","approvedAt","isCrawler","isPublic","mp3size","longitude","trackId","playPath"]
16
+ root = File.expand_path("../..", __FILE__)
17
+ file = File.join(root, 'test/xunch-running.yaml')
18
+ caches = Xunch::CacheBuilder.build(file)
19
+ @object_cache = caches["track"]
20
+ @field_object_cache = caches["trackfield"]
21
+ @list_object_cache = caches["tracklist"]
22
+ @list_field_object_cache = caches["fieldtracklist"]
23
+ @cache_object = TrackRecordOrigin.find(1)
24
+ @cache_objects = [@cache_object]
25
+ @keys = [1]
26
+ for i in 2 .. 100 do
27
+ new_cache_object = TrackRecordOrigin.new
28
+ new_cache_object.track_id = @cache_object.track_id
29
+ new_cache_object.track_uid = @cache_object.track_uid
30
+ new_cache_object.track_upload_source = @cache_object.track_upload_source
31
+ new_cache_object.op_type = @cache_object.op_type
32
+ new_cache_object.is_publish = @cache_object.is_publish
33
+ new_cache_object.upload_source = @cache_object.upload_source
34
+ new_cache_object.uid = @cache_object.uid
35
+ new_cache_object.nickname = @cache_object.nickname
36
+ new_cache_object.avatar_path = @cache_object.avatar_path
37
+ new_cache_object.is_v = @cache_object.is_v
38
+ new_cache_object.human_category_id = @cache_object.human_category_id
39
+ new_cache_object.title = @cache_object.title
40
+ new_cache_object.intro = @cache_object.intro
41
+ new_cache_object.user_source = @cache_object.user_source
42
+ new_cache_object.category_id = @cache_object.category_id
43
+ new_cache_object.duration = @cache_object.duration
44
+ new_cache_object.play_path = @cache_object.play_path
45
+ new_cache_object.play_path_32 = @cache_object.play_path_32
46
+ new_cache_object.play_path_64 = @cache_object.play_path_64
47
+ new_cache_object.play_path_128 = @cache_object.play_path_128
48
+ new_cache_object.transcode_state = @cache_object.transcode_state
49
+ new_cache_object.download_path = @cache_object.download_path
50
+ new_cache_object.cover_path = @cache_object.cover_path
51
+ new_cache_object.album_id = @cache_object.album_id
52
+ new_cache_object.album_title = @cache_object.album_title
53
+ new_cache_object.album_cover_path = @cache_object.album_cover_path
54
+ new_cache_object.tags = @cache_object.tags
55
+ new_cache_object.ignore_tags = @cache_object.ignore_tags
56
+ new_cache_object.extra_tags = @cache_object.extra_tags
57
+ new_cache_object.singer = @cache_object.singer
58
+ new_cache_object.singer_category = @cache_object.singer_category
59
+ new_cache_object.author = @cache_object.author
60
+ new_cache_object.composer = @cache_object.composer
61
+ new_cache_object.arrangement = @cache_object.arrangement
62
+ new_cache_object.post_production = @cache_object.post_production
63
+ new_cache_object.lyric_path = @cache_object.lyric_path
64
+ new_cache_object.lyric = @cache_object.lyric
65
+ new_cache_object.language = @cache_object.language
66
+ new_cache_object.resinger = @cache_object.resinger
67
+ new_cache_object.announcer = @cache_object.announcer
68
+ new_cache_object.is_public = @cache_object.is_public
69
+ new_cache_object.access_password = @cache_object.access_password
70
+ new_cache_object.allow_download = @cache_object.allow_download
71
+ new_cache_object.allow_comment = @cache_object.allow_comment
72
+ new_cache_object.is_crawler = @cache_object.is_crawler
73
+ new_cache_object.inet_aton_ip = @cache_object.inet_aton_ip
74
+ new_cache_object.longitude = @cache_object.longitude
75
+ new_cache_object.latitude = @cache_object.latitude
76
+ new_cache_object.music_category = @cache_object.music_category
77
+ new_cache_object.order_num = @cache_object.order_num
78
+ new_cache_object.is_pick = @cache_object.is_pick
79
+ new_cache_object.rich_intro = @cache_object.rich_intro
80
+ new_cache_object.short_intro = @cache_object.short_intro
81
+ new_cache_object.comment_content = @cache_object.comment_content
82
+ new_cache_object.comment_id = @cache_object.comment_id
83
+ new_cache_object.dig_status = @cache_object.dig_status
84
+ new_cache_object.approved_at = @cache_object.approved_at
85
+ new_cache_object.is_deleted = @cache_object.is_deleted
86
+ new_cache_object.mp3size = @cache_object.mp3size
87
+ new_cache_object.mp3size_32 = @cache_object.mp3size_32
88
+ new_cache_object.mp3size_64 = @cache_object.mp3size_64
89
+ new_cache_object.waveform = @cache_object.waveform
90
+ new_cache_object.upload_id = @cache_object.upload_id
91
+ new_cache_object.updated_at = @cache_object.updated_at
92
+ new_cache_object.created_at = @cache_object.created_at
93
+ new_cache_object.source_url = @cache_object.source_url
94
+ new_cache_object.status = @cache_object.status
95
+ new_cache_object.explore_height = @cache_object.explore_height
96
+ new_cache_object.id = i
97
+ @cache_objects.push new_cache_object
98
+ @keys.push new_cache_object.id
99
+ end
100
+ puts "setup"
101
+ end
102
+
103
+ def test_object_cache_get_put
104
+ # @cache_object.id = 999999
105
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
106
+ @object_cache.put(@cache_object)
107
+ @object_cache.get(@cache_object.id)
108
+ @cache_object.id += 1
109
+ if(@cache_object.id == 9999999999999999999999999999999999999999999999999)
110
+ @cache_object.id = -9999999999999999999999999999999999999999999999999
111
+ end
112
+ puts Time.now
113
+ puts @cache_object.id
114
+ end
115
+ end
116
+
117
+ def test_object_cache_multi_get_put
118
+ # @cache_object.id = 999999
119
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
120
+ keys = []
121
+ @cache_objects.each { | obj |
122
+ puts obj.id
123
+ keys.push obj.id
124
+ }
125
+ @object_cache.multi_put(@cache_objects)
126
+ @object_cache.multi_get(keys)
127
+ @cache_objects.each { | obj |
128
+ obj.id += 100
129
+ }
130
+ puts Time.now
131
+ end
132
+ end
133
+
134
+ def test_field_object_cache_get_put
135
+ # @cache_object.id = 999999
136
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
137
+ @field_object_cache.put_with_field(@cache_object,@fields)
138
+ @field_object_cache.get_with_field(@cache_object.id,@fields)
139
+ @field_object_cache.put(@cache_object)
140
+ @field_object_cache.get(@cache_object.id)
141
+ @cache_object.id += 1
142
+ if(@cache_object.id == 9999999999999999999999999999999999999999999999999)
143
+ @cache_object.id = -9999999999999999999999999999999999999999999999999
144
+ end
145
+ puts Time.now
146
+ puts @cache_object.id
147
+ end
148
+ end
149
+
150
+ def test_field_object_cache_multi_get_put
151
+ # @cache_object.id = 999999
152
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
153
+ keys = []
154
+ @cache_objects.each { | obj |
155
+ puts obj.id
156
+ keys.push obj.id
157
+ }
158
+ @field_object_cache.multi_put_with_field(@cache_objects,@fields)
159
+ @field_object_cache.multi_get_with_field(keys,@fields)
160
+ @field_object_cache.multi_put(@cache_objects)
161
+ @field_object_cache.multi_get(keys)
162
+ @cache_objects.each { | obj |
163
+ obj.id += 100
164
+ }
165
+ puts Time.now
166
+ end
167
+ end
168
+
169
+ def test_list_object_cache_get_put
170
+ # @cache_object.id = 999999
171
+ id = 1
172
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
173
+ keys = []
174
+ @cache_objects.each { | obj |
175
+ puts obj.id
176
+ keys.push obj.id
177
+ }
178
+ @list_object_cache.put(id,@cache_objects)
179
+ @list_object_cache.get(id,1,100)
180
+ @cache_objects.each { | obj |
181
+ obj.id += 100
182
+ }
183
+ puts Time.now
184
+ id+=1
185
+ end
186
+ end
187
+
188
+ def test_list_field_object_cache_get_put
189
+ # @cache_object.id = 999999
190
+ id = 1
191
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
192
+ keys = []
193
+ @cache_objects.each { | obj |
194
+ puts obj.id
195
+ keys.push obj.id
196
+ }
197
+ @list_field_object_cache.put(id,@cache_objects)
198
+ @list_field_object_cache.get(id,1,100)
199
+ @cache_objects.each { | obj |
200
+ obj.id += 100
201
+ }
202
+ puts Time.now
203
+ id+=1
204
+ end
205
+ end
206
+
207
+ def teardown
208
+ super
209
+ puts "teardown"
210
+ end
211
+ end
212
+
@@ -0,0 +1,176 @@
1
+ # require 'date'
2
+ # require 'bigdecimal'
3
+
4
+ # module ModuleA
5
+ # def method_A_1
6
+ # puts "method_A_1"
7
+ # end
8
+ # def attr_type_define(args)
9
+ # args.each { | key, value|
10
+ # case value
11
+ # when :time
12
+ # class_eval("def deserialize_#{key}(args); str_to_time(args); end", __FILE__, __LINE__)
13
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_time))
14
+ # when :bigdecimal
15
+ # class_eval("def deserialize_#{key}(args); str_to_bigdecimal(args); end", __FILE__, __LINE__)
16
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_bigdecimal))
17
+ # when :datetime
18
+ # class_eval("def deserialize_#{key}(args); str_to_datetime(args); end", __FILE__, __LINE__)
19
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_datetime))
20
+ # class_eval("def deserialize_#{key}1(args); no_op(args); end", __FILE__, __LINE__)
21
+ # else
22
+ # class_eval("def deserialize_#{key}(args); no_op(args); end", __FILE__, __LINE__)
23
+ # # define_method("deserialize_" << key.to_s, instance_method(:no_op))
24
+ # end
25
+ # }
26
+ # end
27
+
28
+ # def str_to_time(str)
29
+ # DateTime.strptime(str, "%Y-%m-%dT%H:%M:%S%z").to_time
30
+ # end
31
+ # def str_to_bigdecimal(str)
32
+ # BigDecimal.new(str)
33
+ # end
34
+ # def str_to_datetime(str)
35
+ # DateTime.strptime(str, "%Y-%m-%dT%H:%M:%S%z")
36
+ # end
37
+ # def no_op(value)
38
+ # value
39
+ # end
40
+
41
+ # def self.included(obj)
42
+ # obj.extend(self)
43
+ # end
44
+ # end
45
+ # module ModuleB
46
+ # def method_B_1
47
+ # puts "method_B_1"
48
+ # end
49
+
50
+ # def self.included(obj)
51
+ # obj.extend(self)
52
+ # end
53
+
54
+ # def method_missing(method_id, *args, &block)
55
+ # # puts "def #{method_id}(arg); no_op(arg); end"
56
+ # # class_eval("def deserialize_#{method_id}(*arg); no_op(*arg); end", __FILE__, __LINE__)
57
+ # super
58
+ # end
59
+
60
+ # end
61
+ # class ClassA
62
+ # include ModuleB
63
+ # include ModuleA
64
+ # attr_type_define :column_1 => :time, :column_2 => :bigdecimal, :column_3 => :datetime, :column_4 => :datet
65
+ # end
66
+
67
+ # ClassA.method_A_1
68
+ # ClassA.method_B_1
69
+ # a = ClassA.new
70
+ # ClassA.new.send(:method_B_1)
71
+ # # ClassA.new.send(:m,"1",23){
72
+ # # puts 123
73
+ # # }
74
+
75
+ # puts a.deserialize_column_1("2013-10-23T15:28:51+08:00").class
76
+ # puts a.deserialize_column_2("1312321354543.542342342").class
77
+ # puts a.deserialize_column_3("2013-10-23T15:28:51+08:00").class
78
+ # puts a.deserialize_column_4("2013-10-23T15:28:51+08:00").class
79
+
80
+ # # puts :symbol.id2name
81
+
82
+ # module Test
83
+ # module ClassMethods
84
+
85
+ # end
86
+
87
+ # module InstanceMethods
88
+
89
+ # end
90
+
91
+ # def self.included(receiver)
92
+ # receiver.extend ClassMethods
93
+ # receiver.send :include, InstanceMethods
94
+ # end
95
+ # end
96
+
97
+ # module A
98
+ # class << self
99
+ # attr_reader :logger
100
+ # attr_reader :trace_logger
101
+
102
+ # def trace
103
+ # puts "trace"
104
+ # end
105
+ # end
106
+ # end
107
+
108
+ # class B
109
+ # include A
110
+ # end
111
+
112
+ # goku = Object.new
113
+ # class << goku
114
+ # def power_level
115
+ # p "it's over 9000"
116
+ # end
117
+ # end
118
+ # goku.power_level
119
+
120
+ # puts goku.class
121
+ # puts goku.class != Object
122
+ # raise 'dfdfdf' unless goku.class == Object
123
+
124
+ # puts RUBY_PLATFORM
125
+ # class A
126
+ # TYPE_MAP ={
127
+
128
+ # }
129
+ # end
130
+ # p A.constants
131
+ # p A.const_defined? :TYPE_MAP
132
+ # a = [1].pack('i').size
133
+ # b = a * 8
134
+ # puts a
135
+ # MAX = 2 ** (b - 2) - 1
136
+ # puts MAX
137
+
138
+ # field = "aaa_321_3a_43_dsa_a4"
139
+ # array = field.split("_")
140
+ # puts array
141
+ # field_mod = array[0]
142
+ # # if array.length > 1
143
+ # for i in 1 .. array.length-1 do
144
+ # puts array[i].to_i.to_s
145
+ # puts array[i]
146
+ # # if array[i].to_i.to_s != array[i]
147
+ # field_mod << array[i].capitalize
148
+ # # else
149
+ # # field_mod << array[i]
150
+ # # end
151
+ # # end
152
+ # end
153
+ # puts field_mod.freeze
154
+ # # field_mod <<1
155
+
156
+ # puts "123".capitalize
157
+ # puts "adf123".capitalize
158
+ # puts "1dfd2fd3d".capitalize
159
+ # puts "ad1f2f3ds".capitalize
160
+ # puts "adfd".capitalize
161
+
162
+ # class A
163
+ # class B
164
+ # end
165
+ # end
166
+
167
+ # b = A::B.new
168
+ # p b.class
169
+ # b1 = eval("A::B")
170
+ # p eval("A::B").name
171
+ # p b.class.name
172
+
173
+
174
+ a = " ,".split(",")
175
+ puts a.length
176
+ puts " ".include? ","
@@ -0,0 +1,58 @@
1
+ require 'active_record'
2
+ require 'bigdecimal'
3
+ require 'mysql2'
4
+
5
+ class TrackRecordOrigin < ActiveRecord::Base
6
+ # 声音记录 分表 根据uid > track_records | track_id > track_in_records
7
+
8
+ self.table_name = 'tb_track_record'
9
+ conn = {
10
+ :adapter => "mysql2",
11
+ :encoding => "utf8",
12
+ :reconnect => false,
13
+ :database => "test",
14
+ :username => "root",
15
+ :password => "111111",
16
+ :host => "127.0.0.1"
17
+ }
18
+ ActiveRecord::Base.establish_connection(conn)
19
+ attr_accessible :track_id,
20
+ :track_uid,
21
+ :track_upload_source,
22
+ :op_type,
23
+ :is_publish,
24
+ :upload_source,
25
+ :uid, :nickname, :avatar_path, :is_v, :human_category_id,
26
+ :title, :intro,
27
+ :user_source,
28
+ :category_id,
29
+ :duration,
30
+ :play_path, :play_path_32, :play_path_64, :play_path_128,
31
+ :transcode_state,
32
+ :download_path,
33
+ :cover_path,
34
+ :album_id, :album_title,
35
+ :album_cover_path,
36
+ :tags, :ignore_tags, :extra_tags,
37
+ :singer, :singer_category, :author,
38
+ :composer, :arrangement, :post_production,
39
+ :lyric, :lyric_path,
40
+ :language, :resinger, :announcer,
41
+ :is_public, :access_password,
42
+ :allow_download, :allow_comment,
43
+ :is_crawler, :inet_aton_ip, :longitude, :latitude,
44
+ :music_category, :order_num, :is_pick,
45
+ :rich_intro, :short_intro, :comment_content, :comment_id,
46
+ :dig_status, :approved_at,
47
+ :is_deleted,
48
+ :mp3size, :mp3size_32, :mp3size_64, :waveform,
49
+ :upload_id,
50
+ :source_url, :status, :explore_height
51
+
52
+
53
+ # op_type --- 1:上传、更新/2:转发(根据原track产生又一条record记录)
54
+ OP_TYPE = {UPLOAD:1, RELAY:2}
55
+
56
+ # comment_content --- 转发时输入的评论
57
+
58
+ end