xunch 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/lib/xunch.rb +25 -0
  3. data/lib/xunch/cache/cache.rb +88 -0
  4. data/lib/xunch/cache/cache_builder.rb +68 -0
  5. data/lib/xunch/cache/field_object_cache.rb +120 -0
  6. data/lib/xunch/cache/list_field_object_cache.rb +63 -0
  7. data/lib/xunch/cache/list_object_cache.rb +59 -0
  8. data/lib/xunch/cache/object_cache.rb +63 -0
  9. data/lib/xunch/codec/codec.rb +31 -0
  10. data/lib/xunch/codec/hash_codec.rb +98 -0
  11. data/lib/xunch/codec/json_codec.rb +81 -0
  12. data/lib/xunch/shard/redis.rb +270 -0
  13. data/lib/xunch/shard/shard_info.rb +37 -0
  14. data/lib/xunch/shard/shard_redis.rb +267 -0
  15. data/lib/xunch/shard/sharded.rb +50 -0
  16. data/lib/xunch/utils/exceptions.rb +11 -0
  17. data/lib/xunch/utils/nginx_cache_helper.rb +52 -0
  18. data/lib/xunch/utils/rb_tree.rb +634 -0
  19. data/lib/xunch/utils/rb_tree_node.rb +67 -0
  20. data/lib/xunch/utils/types.rb +8 -0
  21. data/lib/xunch/utils/utils.rb +24 -0
  22. data/test/benchmark_test.rb +68 -0
  23. data/test/cache_builder_test.rb +28 -0
  24. data/test/cache_object.rb +120 -0
  25. data/test/consistency_hash_test.rb +31 -0
  26. data/test/field_object_cache_test.rb +430 -0
  27. data/test/hash_codec_test.rb +57 -0
  28. data/test/json_codec_test.rb +57 -0
  29. data/test/list_field_object_cache_test.rb +211 -0
  30. data/test/list_object_cache_test.rb +211 -0
  31. data/test/nginx_cache_helper_test.rb +45 -0
  32. data/test/object_cache_test.rb +322 -0
  33. data/test/rb_tree_test.rb +48 -0
  34. data/test/redis_benchmark_test.rb +54 -0
  35. data/test/redis_test.rb +58 -0
  36. data/test/running_test.rb +212 -0
  37. data/test/test.rb +176 -0
  38. data/test/track_record_origin.rb +58 -0
  39. metadata +125 -0
@@ -0,0 +1,58 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+
3
+ require "test/unit"
4
+ require "xunch"
5
+
6
+ class RedisTest < Test::Unit::TestCase
7
+ include Test::Unit::Assertions
8
+ def setup
9
+ super
10
+ options = {
11
+ :host => "192.168.1.174",
12
+ :port => 6379,
13
+ :timeout => 5.0,
14
+ :password => 'jredis123456',
15
+ :db => 10,
16
+ :pool_timeout => 12,
17
+ :size => 10
18
+ }
19
+ @redis_client = Xunch::RedisClient.new(options)
20
+ puts "setup"
21
+ end
22
+
23
+ def test_redis
24
+ assert_equal('OK',@redis_client.set('key1','value1',100000))
25
+ assert(@redis_client.exists('key1'),"expect exists key1")
26
+ assert((@redis_client.ttl('key1')<100000),"expect ttl less than 100000")
27
+ assert(@redis_client.expire('key1',1000000))
28
+ assert(@redis_client.ttl('key1')>100000,"expect ttl more than 100000")
29
+ assert(@redis_client.ttl('key1')<1000000,"expect ttl less than 1000000")
30
+ assert_equal('value1',@redis_client.get('key1'))
31
+ assert(@redis_client.del('key1'),"del key1 didn't success")
32
+
33
+ hash = {"k1"=>"v1","k2"=>"v2","k3"=>"v3","k4"=>"v4","k5"=>"v5"}
34
+ assert_equal(["OK", true, true, true, true, true],@redis_client.mset(hash,1000000))
35
+
36
+ assert_equal(["v1", "v2", "v3", "v4", "v5", nil],@redis_client.mget("k1","k2","k3","k4","k5","k6"))
37
+
38
+ fields = {"field1"=>"v1","field2"=>"v2","field3"=>"v3","field4"=>"v4","field5"=>"v5"}
39
+ fieldkeys = ["field1","field2","field3","field4","field5"]
40
+
41
+ assert_equal(["OK", true], @redis_client.hset("hash_key1",fields,10000))
42
+ assert_equal({"field1"=>"v1", "field2"=>"v2", "field3"=>"v3", "field4"=>"v4", "field5"=>"v5"}, @redis_client.hget("hash_key1",*fieldkeys))
43
+ assert_equal({"field1"=>"v1", "field2"=>"v2", "field3"=>"v3", "field4"=>"v4", "field5"=>"v5"}, @redis_client.hgetall("hash_key1"))
44
+
45
+ hash = {"hash_k1"=>fields,"hash_k2"=>fields,"hash_k3"=>fields,"hash_k4"=>fields,"hash_k5"=>fields}
46
+ hashkeys =["hash_k1","hash_k2","hash_k0","hash_k8","hash_k5"]
47
+ assert_equal(["OK", true, "OK", true, "OK", true, "OK", true, "OK", true], @redis_client.hmset(hash,100000))
48
+
49
+ @redis_client.hmget(hashkeys,*fieldkeys)
50
+
51
+ end
52
+
53
+ def teardown
54
+ super
55
+ @redis_client.destroy
56
+ puts "teardown"
57
+ end
58
+ end
@@ -0,0 +1,212 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+ $:.unshift File.expand_path("../../test", __FILE__)
3
+ require "test/unit"
4
+ require "xunch"
5
+ require 'yaml'
6
+ require 'cache_object'
7
+ require 'bigdecimal'
8
+ require 'hessian2'
9
+ require 'track_record_origin'
10
+
11
+ class RunningCacheTest < Test::Unit::TestCase
12
+ include Test::Unit::Assertions
13
+ def setup
14
+ super
15
+ @fields = ["createdAt","updatedAt","approvedAt","isCrawler","isPublic","mp3size","longitude","trackId","playPath"]
16
+ root = File.expand_path("../..", __FILE__)
17
+ file = File.join(root, 'test/xunch-running.yaml')
18
+ caches = Xunch::CacheBuilder.build(file)
19
+ @object_cache = caches["track"]
20
+ @field_object_cache = caches["trackfield"]
21
+ @list_object_cache = caches["tracklist"]
22
+ @list_field_object_cache = caches["fieldtracklist"]
23
+ @cache_object = TrackRecordOrigin.find(1)
24
+ @cache_objects = [@cache_object]
25
+ @keys = [1]
26
+ for i in 2 .. 100 do
27
+ new_cache_object = TrackRecordOrigin.new
28
+ new_cache_object.track_id = @cache_object.track_id
29
+ new_cache_object.track_uid = @cache_object.track_uid
30
+ new_cache_object.track_upload_source = @cache_object.track_upload_source
31
+ new_cache_object.op_type = @cache_object.op_type
32
+ new_cache_object.is_publish = @cache_object.is_publish
33
+ new_cache_object.upload_source = @cache_object.upload_source
34
+ new_cache_object.uid = @cache_object.uid
35
+ new_cache_object.nickname = @cache_object.nickname
36
+ new_cache_object.avatar_path = @cache_object.avatar_path
37
+ new_cache_object.is_v = @cache_object.is_v
38
+ new_cache_object.human_category_id = @cache_object.human_category_id
39
+ new_cache_object.title = @cache_object.title
40
+ new_cache_object.intro = @cache_object.intro
41
+ new_cache_object.user_source = @cache_object.user_source
42
+ new_cache_object.category_id = @cache_object.category_id
43
+ new_cache_object.duration = @cache_object.duration
44
+ new_cache_object.play_path = @cache_object.play_path
45
+ new_cache_object.play_path_32 = @cache_object.play_path_32
46
+ new_cache_object.play_path_64 = @cache_object.play_path_64
47
+ new_cache_object.play_path_128 = @cache_object.play_path_128
48
+ new_cache_object.transcode_state = @cache_object.transcode_state
49
+ new_cache_object.download_path = @cache_object.download_path
50
+ new_cache_object.cover_path = @cache_object.cover_path
51
+ new_cache_object.album_id = @cache_object.album_id
52
+ new_cache_object.album_title = @cache_object.album_title
53
+ new_cache_object.album_cover_path = @cache_object.album_cover_path
54
+ new_cache_object.tags = @cache_object.tags
55
+ new_cache_object.ignore_tags = @cache_object.ignore_tags
56
+ new_cache_object.extra_tags = @cache_object.extra_tags
57
+ new_cache_object.singer = @cache_object.singer
58
+ new_cache_object.singer_category = @cache_object.singer_category
59
+ new_cache_object.author = @cache_object.author
60
+ new_cache_object.composer = @cache_object.composer
61
+ new_cache_object.arrangement = @cache_object.arrangement
62
+ new_cache_object.post_production = @cache_object.post_production
63
+ new_cache_object.lyric_path = @cache_object.lyric_path
64
+ new_cache_object.lyric = @cache_object.lyric
65
+ new_cache_object.language = @cache_object.language
66
+ new_cache_object.resinger = @cache_object.resinger
67
+ new_cache_object.announcer = @cache_object.announcer
68
+ new_cache_object.is_public = @cache_object.is_public
69
+ new_cache_object.access_password = @cache_object.access_password
70
+ new_cache_object.allow_download = @cache_object.allow_download
71
+ new_cache_object.allow_comment = @cache_object.allow_comment
72
+ new_cache_object.is_crawler = @cache_object.is_crawler
73
+ new_cache_object.inet_aton_ip = @cache_object.inet_aton_ip
74
+ new_cache_object.longitude = @cache_object.longitude
75
+ new_cache_object.latitude = @cache_object.latitude
76
+ new_cache_object.music_category = @cache_object.music_category
77
+ new_cache_object.order_num = @cache_object.order_num
78
+ new_cache_object.is_pick = @cache_object.is_pick
79
+ new_cache_object.rich_intro = @cache_object.rich_intro
80
+ new_cache_object.short_intro = @cache_object.short_intro
81
+ new_cache_object.comment_content = @cache_object.comment_content
82
+ new_cache_object.comment_id = @cache_object.comment_id
83
+ new_cache_object.dig_status = @cache_object.dig_status
84
+ new_cache_object.approved_at = @cache_object.approved_at
85
+ new_cache_object.is_deleted = @cache_object.is_deleted
86
+ new_cache_object.mp3size = @cache_object.mp3size
87
+ new_cache_object.mp3size_32 = @cache_object.mp3size_32
88
+ new_cache_object.mp3size_64 = @cache_object.mp3size_64
89
+ new_cache_object.waveform = @cache_object.waveform
90
+ new_cache_object.upload_id = @cache_object.upload_id
91
+ new_cache_object.updated_at = @cache_object.updated_at
92
+ new_cache_object.created_at = @cache_object.created_at
93
+ new_cache_object.source_url = @cache_object.source_url
94
+ new_cache_object.status = @cache_object.status
95
+ new_cache_object.explore_height = @cache_object.explore_height
96
+ new_cache_object.id = i
97
+ @cache_objects.push new_cache_object
98
+ @keys.push new_cache_object.id
99
+ end
100
+ puts "setup"
101
+ end
102
+
103
+ def test_object_cache_get_put
104
+ # @cache_object.id = 999999
105
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
106
+ @object_cache.put(@cache_object)
107
+ @object_cache.get(@cache_object.id)
108
+ @cache_object.id += 1
109
+ if(@cache_object.id == 9999999999999999999999999999999999999999999999999)
110
+ @cache_object.id = -9999999999999999999999999999999999999999999999999
111
+ end
112
+ puts Time.now
113
+ puts @cache_object.id
114
+ end
115
+ end
116
+
117
+ def test_object_cache_multi_get_put
118
+ # @cache_object.id = 999999
119
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
120
+ keys = []
121
+ @cache_objects.each { | obj |
122
+ puts obj.id
123
+ keys.push obj.id
124
+ }
125
+ @object_cache.multi_put(@cache_objects)
126
+ @object_cache.multi_get(keys)
127
+ @cache_objects.each { | obj |
128
+ obj.id += 100
129
+ }
130
+ puts Time.now
131
+ end
132
+ end
133
+
134
+ def test_field_object_cache_get_put
135
+ # @cache_object.id = 999999
136
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
137
+ @field_object_cache.put_with_field(@cache_object,@fields)
138
+ @field_object_cache.get_with_field(@cache_object.id,@fields)
139
+ @field_object_cache.put(@cache_object)
140
+ @field_object_cache.get(@cache_object.id)
141
+ @cache_object.id += 1
142
+ if(@cache_object.id == 9999999999999999999999999999999999999999999999999)
143
+ @cache_object.id = -9999999999999999999999999999999999999999999999999
144
+ end
145
+ puts Time.now
146
+ puts @cache_object.id
147
+ end
148
+ end
149
+
150
+ def test_field_object_cache_multi_get_put
151
+ # @cache_object.id = 999999
152
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
153
+ keys = []
154
+ @cache_objects.each { | obj |
155
+ puts obj.id
156
+ keys.push obj.id
157
+ }
158
+ @field_object_cache.multi_put_with_field(@cache_objects,@fields)
159
+ @field_object_cache.multi_get_with_field(keys,@fields)
160
+ @field_object_cache.multi_put(@cache_objects)
161
+ @field_object_cache.multi_get(keys)
162
+ @cache_objects.each { | obj |
163
+ obj.id += 100
164
+ }
165
+ puts Time.now
166
+ end
167
+ end
168
+
169
+ def test_list_object_cache_get_put
170
+ # @cache_object.id = 999999
171
+ id = 1
172
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
173
+ keys = []
174
+ @cache_objects.each { | obj |
175
+ puts obj.id
176
+ keys.push obj.id
177
+ }
178
+ @list_object_cache.put(id,@cache_objects)
179
+ @list_object_cache.get(id,1,100)
180
+ @cache_objects.each { | obj |
181
+ obj.id += 100
182
+ }
183
+ puts Time.now
184
+ id+=1
185
+ end
186
+ end
187
+
188
+ def test_list_field_object_cache_get_put
189
+ # @cache_object.id = 999999
190
+ id = 1
191
+ while @cache_object.id < 9999999999999999999999999999999999999999999999999
192
+ keys = []
193
+ @cache_objects.each { | obj |
194
+ puts obj.id
195
+ keys.push obj.id
196
+ }
197
+ @list_field_object_cache.put(id,@cache_objects)
198
+ @list_field_object_cache.get(id,1,100)
199
+ @cache_objects.each { | obj |
200
+ obj.id += 100
201
+ }
202
+ puts Time.now
203
+ id+=1
204
+ end
205
+ end
206
+
207
+ def teardown
208
+ super
209
+ puts "teardown"
210
+ end
211
+ end
212
+
@@ -0,0 +1,176 @@
1
+ # require 'date'
2
+ # require 'bigdecimal'
3
+
4
+ # module ModuleA
5
+ # def method_A_1
6
+ # puts "method_A_1"
7
+ # end
8
+ # def attr_type_define(args)
9
+ # args.each { | key, value|
10
+ # case value
11
+ # when :time
12
+ # class_eval("def deserialize_#{key}(args); str_to_time(args); end", __FILE__, __LINE__)
13
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_time))
14
+ # when :bigdecimal
15
+ # class_eval("def deserialize_#{key}(args); str_to_bigdecimal(args); end", __FILE__, __LINE__)
16
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_bigdecimal))
17
+ # when :datetime
18
+ # class_eval("def deserialize_#{key}(args); str_to_datetime(args); end", __FILE__, __LINE__)
19
+ # # define_method("deserialize_" << key.to_s, instance_method(:str_to_datetime))
20
+ # class_eval("def deserialize_#{key}1(args); no_op(args); end", __FILE__, __LINE__)
21
+ # else
22
+ # class_eval("def deserialize_#{key}(args); no_op(args); end", __FILE__, __LINE__)
23
+ # # define_method("deserialize_" << key.to_s, instance_method(:no_op))
24
+ # end
25
+ # }
26
+ # end
27
+
28
+ # def str_to_time(str)
29
+ # DateTime.strptime(str, "%Y-%m-%dT%H:%M:%S%z").to_time
30
+ # end
31
+ # def str_to_bigdecimal(str)
32
+ # BigDecimal.new(str)
33
+ # end
34
+ # def str_to_datetime(str)
35
+ # DateTime.strptime(str, "%Y-%m-%dT%H:%M:%S%z")
36
+ # end
37
+ # def no_op(value)
38
+ # value
39
+ # end
40
+
41
+ # def self.included(obj)
42
+ # obj.extend(self)
43
+ # end
44
+ # end
45
+ # module ModuleB
46
+ # def method_B_1
47
+ # puts "method_B_1"
48
+ # end
49
+
50
+ # def self.included(obj)
51
+ # obj.extend(self)
52
+ # end
53
+
54
+ # def method_missing(method_id, *args, &block)
55
+ # # puts "def #{method_id}(arg); no_op(arg); end"
56
+ # # class_eval("def deserialize_#{method_id}(*arg); no_op(*arg); end", __FILE__, __LINE__)
57
+ # super
58
+ # end
59
+
60
+ # end
61
+ # class ClassA
62
+ # include ModuleB
63
+ # include ModuleA
64
+ # attr_type_define :column_1 => :time, :column_2 => :bigdecimal, :column_3 => :datetime, :column_4 => :datet
65
+ # end
66
+
67
+ # ClassA.method_A_1
68
+ # ClassA.method_B_1
69
+ # a = ClassA.new
70
+ # ClassA.new.send(:method_B_1)
71
+ # # ClassA.new.send(:m,"1",23){
72
+ # # puts 123
73
+ # # }
74
+
75
+ # puts a.deserialize_column_1("2013-10-23T15:28:51+08:00").class
76
+ # puts a.deserialize_column_2("1312321354543.542342342").class
77
+ # puts a.deserialize_column_3("2013-10-23T15:28:51+08:00").class
78
+ # puts a.deserialize_column_4("2013-10-23T15:28:51+08:00").class
79
+
80
+ # # puts :symbol.id2name
81
+
82
+ # module Test
83
+ # module ClassMethods
84
+
85
+ # end
86
+
87
+ # module InstanceMethods
88
+
89
+ # end
90
+
91
+ # def self.included(receiver)
92
+ # receiver.extend ClassMethods
93
+ # receiver.send :include, InstanceMethods
94
+ # end
95
+ # end
96
+
97
+ # module A
98
+ # class << self
99
+ # attr_reader :logger
100
+ # attr_reader :trace_logger
101
+
102
+ # def trace
103
+ # puts "trace"
104
+ # end
105
+ # end
106
+ # end
107
+
108
+ # class B
109
+ # include A
110
+ # end
111
+
112
+ # goku = Object.new
113
+ # class << goku
114
+ # def power_level
115
+ # p "it's over 9000"
116
+ # end
117
+ # end
118
+ # goku.power_level
119
+
120
+ # puts goku.class
121
+ # puts goku.class != Object
122
+ # raise 'dfdfdf' unless goku.class == Object
123
+
124
+ # puts RUBY_PLATFORM
125
+ # class A
126
+ # TYPE_MAP ={
127
+
128
+ # }
129
+ # end
130
+ # p A.constants
131
+ # p A.const_defined? :TYPE_MAP
132
+ # a = [1].pack('i').size
133
+ # b = a * 8
134
+ # puts a
135
+ # MAX = 2 ** (b - 2) - 1
136
+ # puts MAX
137
+
138
+ # field = "aaa_321_3a_43_dsa_a4"
139
+ # array = field.split("_")
140
+ # puts array
141
+ # field_mod = array[0]
142
+ # # if array.length > 1
143
+ # for i in 1 .. array.length-1 do
144
+ # puts array[i].to_i.to_s
145
+ # puts array[i]
146
+ # # if array[i].to_i.to_s != array[i]
147
+ # field_mod << array[i].capitalize
148
+ # # else
149
+ # # field_mod << array[i]
150
+ # # end
151
+ # # end
152
+ # end
153
+ # puts field_mod.freeze
154
+ # # field_mod <<1
155
+
156
+ # puts "123".capitalize
157
+ # puts "adf123".capitalize
158
+ # puts "1dfd2fd3d".capitalize
159
+ # puts "ad1f2f3ds".capitalize
160
+ # puts "adfd".capitalize
161
+
162
+ # class A
163
+ # class B
164
+ # end
165
+ # end
166
+
167
+ # b = A::B.new
168
+ # p b.class
169
+ # b1 = eval("A::B")
170
+ # p eval("A::B").name
171
+ # p b.class.name
172
+
173
+
174
+ a = " ,".split(",")
175
+ puts a.length
176
+ puts " ".include? ","
@@ -0,0 +1,58 @@
1
+ require 'active_record'
2
+ require 'bigdecimal'
3
+ require 'mysql2'
4
+
5
+ class TrackRecordOrigin < ActiveRecord::Base
6
+ # 声音记录 分表 根据uid > track_records | track_id > track_in_records
7
+
8
+ self.table_name = 'tb_track_record'
9
+ conn = {
10
+ :adapter => "mysql2",
11
+ :encoding => "utf8",
12
+ :reconnect => false,
13
+ :database => "test",
14
+ :username => "root",
15
+ :password => "111111",
16
+ :host => "127.0.0.1"
17
+ }
18
+ ActiveRecord::Base.establish_connection(conn)
19
+ attr_accessible :track_id,
20
+ :track_uid,
21
+ :track_upload_source,
22
+ :op_type,
23
+ :is_publish,
24
+ :upload_source,
25
+ :uid, :nickname, :avatar_path, :is_v, :human_category_id,
26
+ :title, :intro,
27
+ :user_source,
28
+ :category_id,
29
+ :duration,
30
+ :play_path, :play_path_32, :play_path_64, :play_path_128,
31
+ :transcode_state,
32
+ :download_path,
33
+ :cover_path,
34
+ :album_id, :album_title,
35
+ :album_cover_path,
36
+ :tags, :ignore_tags, :extra_tags,
37
+ :singer, :singer_category, :author,
38
+ :composer, :arrangement, :post_production,
39
+ :lyric, :lyric_path,
40
+ :language, :resinger, :announcer,
41
+ :is_public, :access_password,
42
+ :allow_download, :allow_comment,
43
+ :is_crawler, :inet_aton_ip, :longitude, :latitude,
44
+ :music_category, :order_num, :is_pick,
45
+ :rich_intro, :short_intro, :comment_content, :comment_id,
46
+ :dig_status, :approved_at,
47
+ :is_deleted,
48
+ :mp3size, :mp3size_32, :mp3size_64, :waveform,
49
+ :upload_id,
50
+ :source_url, :status, :explore_height
51
+
52
+
53
+ # op_type --- 1:上传、更新/2:转发(根据原track产生又一条record记录)
54
+ OP_TYPE = {UPLOAD:1, RELAY:2}
55
+
56
+ # comment_content --- 转发时输入的评论
57
+
58
+ end