roma 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/LICENSE.rdoc +675 -0
  2. data/README.rdoc +0 -0
  3. data/Rakefile +70 -0
  4. data/bin/mkrecent +7 -0
  5. data/bin/mkroute +7 -0
  6. data/bin/recoverlost +8 -0
  7. data/bin/recoverlost_alist +8 -0
  8. data/bin/romad +7 -0
  9. data/bin/sample_watcher +8 -0
  10. data/bin/sample_watcher2 +8 -0
  11. data/bin/simple_bench +8 -0
  12. data/bin/ssroute +7 -0
  13. data/bin/tribunus +7 -0
  14. data/lib/roma/async_process.rb +696 -0
  15. data/lib/roma/command/bg_command_receiver.rb +188 -0
  16. data/lib/roma/command/mh_command_receiver.rb +117 -0
  17. data/lib/roma/command/receiver.rb +287 -0
  18. data/lib/roma/command/rt_command_receiver.rb +147 -0
  19. data/lib/roma/command/st_command_receiver.rb +564 -0
  20. data/lib/roma/command/util_command_receiver.rb +67 -0
  21. data/lib/roma/command/vn_command_receiver.rb +143 -0
  22. data/lib/roma/command_plugin.rb +11 -0
  23. data/lib/roma/config.rb +64 -0
  24. data/lib/roma/event/con_pool.rb +140 -0
  25. data/lib/roma/event/handler.rb +159 -0
  26. data/lib/roma/plugin/plugin_alist.rb +1572 -0
  27. data/lib/roma/plugin/plugin_debug.rb +19 -0
  28. data/lib/roma/plugin/plugin_test.rb +14 -0
  29. data/lib/roma/romad.rb +582 -0
  30. data/lib/roma/routing/cb_rttable.rb +326 -0
  31. data/lib/roma/routing/merkle_tree.rb +54 -0
  32. data/lib/roma/routing/rttable.rb +148 -0
  33. data/lib/roma/stats.rb +112 -0
  34. data/lib/roma/storage/basic_storage.rb +510 -0
  35. data/lib/roma/storage/dbm_storage.rb +80 -0
  36. data/lib/roma/storage/dummy_storage.rb +44 -0
  37. data/lib/roma/storage/rh_storage.rb +35 -0
  38. data/lib/roma/storage/sqlite3_storage.rb +73 -0
  39. data/lib/roma/storage/tc_storage.rb +133 -0
  40. data/lib/roma/tools/mkrecent.rb +138 -0
  41. data/lib/roma/tools/mkroute.rb +52 -0
  42. data/lib/roma/tools/recoverlost.rb +9 -0
  43. data/lib/roma/tools/recoverlost_alist.rb +9 -0
  44. data/lib/roma/tools/recoverlost_lib.rb +217 -0
  45. data/lib/roma/tools/sample_watcher.rb +38 -0
  46. data/lib/roma/tools/sample_watcher2.rb +38 -0
  47. data/lib/roma/tools/simple_bench.rb +57 -0
  48. data/lib/roma/tools/ssroute.rb +23 -0
  49. data/lib/roma/tools/tribunus.rb +299 -0
  50. data/lib/roma/version.rb +4 -0
  51. data/lib/roma/write_behind.rb +179 -0
  52. data/test/rcirb.rb +16 -0
  53. data/test/roma-test-utils.rb +65 -0
  54. data/test/run-test.rb +16 -0
  55. data/test/t_cpdata.rb +277 -0
  56. data/test/t_listplugin.rb +592 -0
  57. data/test/t_rclient.rb +318 -0
  58. data/test/t_routing_data.rb +100 -0
  59. data/test/t_storage.rb +644 -0
  60. data/test/t_writebehind.rb +200 -0
  61. metadata +134 -0
data/test/t_cpdata.rb ADDED
@@ -0,0 +1,277 @@
1
+ #!/usr/bin/env ruby
2
+ # -*- coding: utf-8 -*-
3
+
4
+ require 'roma/client/rclient'
5
+
6
+ $dat = {}
7
+
8
+ def receive_command_server
9
+ $gs = TCPServer.open(11213)
10
+ while true
11
+ Thread.new($gs.accept){|s|
12
+ begin
13
+ loop {
14
+ res = s.gets
15
+ p res
16
+ if res==nil
17
+ s.close
18
+ elsif res.start_with?("pushv")
19
+ ss = res.split(" ")
20
+ s.write("READY\r\n")
21
+ len = s.gets.chomp
22
+ $dat[ss[2].to_i] = receive_dump(s, len.to_i)
23
+ s.write("STORED\r\n")
24
+ elsif res.start_with?("spushv")
25
+ ss = res.split(" ")
26
+ s.write("READY\r\n")
27
+ $dat[ss[2].to_i] = receive_stream_dump(s)
28
+ s.write("STORED\r\n")
29
+ elsif res.start_with?("whoami")
30
+ s.write("ROMA\r\n")
31
+ elsif res.start_with?("rbalse")
32
+ s.write("BYE\r\n")
33
+ s.close
34
+ break
35
+ else
36
+ s.write("STORED\r\n")
37
+ end
38
+ }
39
+ rescue =>e
40
+ p e
41
+ p $@
42
+ end
43
+ }
44
+ end
45
+ rescue =>e
46
+ p e
47
+ end
48
+
49
+ def receive_stream_dump(sok)
50
+ ret = {}
51
+ v = nil
52
+ loop {
53
+ context_bin = sok.read(20)
54
+ vn, last, clk, expt, klen = context_bin.unpack('NNNNN')
55
+
56
+ break if klen == 0 # end of dump ?
57
+ k = sok.read(klen)
58
+ vlen_bin = sok.read(4)
59
+ vlen, = vlen_bin.unpack('N')
60
+ if vlen != 0
61
+ v = sok.read(vlen)
62
+ end
63
+ ret[k] = [vn, last, clk, expt, v].pack('NNNNa*')
64
+ }
65
+ ret
66
+ rescue =>e
67
+ p e
68
+ end
69
+
70
+ def receive_dump(sok, len)
71
+ dmp = ''
72
+ while(dmp.length != len.to_i)
73
+ dmp = dmp + sok.read(len.to_i - dmp.length)
74
+ end
75
+ sok.read(2)
76
+ if sok.gets == "END\r\n"
77
+ return Marshal.load(dmp)
78
+ else
79
+ return nil
80
+ end
81
+ rescue =>e
82
+ false
83
+ end
84
+
85
+ # vnode をコピーするテスト
86
+ class CopyDataTest < Test::Unit::TestCase
87
+ include RomaTestUtils
88
+
89
+ def setup
90
+ @th = Thread.new{ receive_command_server }
91
+ start_roma
92
+ @rc=Roma::Client::RomaClient.new(["localhost_11211","localhost_11212"])
93
+ end
94
+
95
+ def teardown
96
+ stop_roma
97
+ @th.kill
98
+ $gs.close
99
+ Roma::Messaging::ConPool::instance.close_all
100
+ end
101
+
102
+ def test_pushv
103
+ make_dummy(1000)
104
+ dat = reqpushv('roma',0)
105
+ assert_not_nil( dat )
106
+ # 正常ケース
107
+ con = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
108
+ assert_equal("STORED", push_a_vnode('roma',0,con,Marshal.dump(dat)))
109
+
110
+ # 存在しない仮想ストレージ
111
+ con = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
112
+ assert_equal("SERVER_ERROR @storages[roma1] dose not found.",
113
+ push_a_vnode('roma1',0,con,Marshal.dump(dat)))
114
+
115
+ # END を送らない
116
+ con = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
117
+ assert_equal("SERVER_ERROR END was not able to be received.",
118
+ push_a_vnode('roma',0,con,Marshal.dump(dat),true))
119
+
120
+ # 壊れたデータを送る
121
+ dat['abc']="ajjkdlfsoifulwkejrweorlkjflksjflskaf"
122
+ con = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
123
+ assert_equal(
124
+ "SERVER_ERROR An invalid vnode number is include.key=abc vn=1634364011",
125
+ push_a_vnode('roma',2,con,Marshal.dump(dat)))
126
+ end
127
+
128
+ def push_a_vnode(hname ,vn, con, dmp, nonend=false)
129
+ con.write("pushv #{hname} #{vn}\r\n")
130
+ res = con.gets # READY\r\n or error string
131
+ if res != "READY\r\n"
132
+ con.close
133
+ return res.chomp
134
+ end
135
+ if nonend
136
+ con.write("#{dmp.length}\r\n#{dmp}\r\n\r\n")
137
+ else
138
+ con.write("#{dmp.length}\r\n#{dmp}\r\nEND\r\n")
139
+ end
140
+ res = con.gets # STORED\r\n or error string
141
+ con.close
142
+ res.chomp! if res
143
+ res
144
+ rescue =>e
145
+ con.close if con
146
+ "#{e}"
147
+ end
148
+ private :push_a_vnode
149
+
150
+
151
+ def test_spushv
152
+ # vn = 0 のキー
153
+ keys = []
154
+ n = 1000
155
+ n.times{|i|
156
+ d = Digest::SHA1.hexdigest(i.to_s).hex % @rc.rttable.hbits
157
+ vn = @rc.rttable.get_vnode_id(d)
158
+ if vn == 0
159
+ keys << i.to_s
160
+ end
161
+ }
162
+ nid = @rc.rttable.search_nodes(0)
163
+
164
+ push_a_vnode_stream('roma', 0, nid[0], keys)
165
+
166
+ keys.each{|k|
167
+ assert_equal( "#{k}-stream", @rc.get(k))
168
+ # puts "#{k} #{@rc.get(k)}"
169
+ }
170
+ end
171
+
172
+ def push_a_vnode_stream(hname, vn, nid, keys)
173
+ con = Roma::Messaging::ConPool.instance.get_connection(nid)
174
+ con.write("spushv #{hname} #{vn}\r\n")
175
+
176
+ res = con.gets # READY\r\n or error string
177
+ if res != "READY\r\n"
178
+ con.close
179
+ return res.chomp
180
+ end
181
+
182
+ keys.each{|k|
183
+ v = k + "-stream"
184
+ data = [vn, Time.now.to_i, 1, 0x7fffffff, k.length, k, v.length, v].pack("NNNNNa#{k.length}Na#{v.length}")
185
+ con.write(data)
186
+ }
187
+ con.write("\0"*20) # end of steram
188
+
189
+ res = con.gets # STORED\r\n or error string
190
+ Roma::Messaging::ConPool.instance.return_connection(nid,con)
191
+ res.chomp! if res
192
+ res
193
+ rescue =>e
194
+ "#{e}"
195
+ end
196
+ private :push_a_vnode_stream
197
+
198
+
199
+ def test_reqpushv
200
+ make_dummy(1000)
201
+
202
+ dat=[]
203
+ dat[0] = reqpushv('roma',0)
204
+ assert_not_nil( dat[0] )
205
+ dat[0] = reqpushv('roma',0)
206
+ assert_not_nil( dat[0] ) # 同じ vnode を2度アクセスしても良いことを確認
207
+
208
+ dat[1] = reqpushv('roma',536870912)
209
+ assert_not_nil( dat[1] )
210
+ dat[2] = reqpushv('roma',1073741824)
211
+ assert_not_nil( dat[2] )
212
+ dat[3] = reqpushv('roma',1610612736)
213
+ assert_not_nil( dat[3])
214
+ dat[4] = reqpushv('roma',2147483648)
215
+ assert_not_nil( dat[4] )
216
+ dat[5] = reqpushv('roma',2684354560)
217
+ assert_not_nil( dat[5] )
218
+ dat[6] = reqpushv('roma',3221225472,true)
219
+ assert_not_nil( dat[6] )
220
+ dat[7] = reqpushv('roma',3758096384,true)
221
+ assert_not_nil( dat[7] )
222
+
223
+ a = 0
224
+ dat.each{|v| a+=v.length }
225
+ assert_equal( 1000,a )
226
+ end
227
+
228
+ def wait(vn)
229
+ while $dat.key?(vn) do
230
+ sleep 0.01
231
+ end
232
+ $dat[vn]
233
+ end
234
+
235
+ # n 個の dummy data を set
236
+ def make_dummy(n)
237
+ n.times{|i|
238
+ assert( @rc.set(i.to_s,i.to_s)=="STORED" )
239
+ }
240
+ end
241
+
242
+ def reqpushv(hname,vn,is_primary=false)
243
+ $dat.delete(vn)
244
+ con = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
245
+ con.write("reqpushv #{vn} localhost_11213 #{is_primary}\r\n")
246
+ res = con.gets
247
+ assert_equal( "PUSHED\r\n", res )
248
+ con.close
249
+
250
+ until $dat.key?(vn) do
251
+ Thread.pass
252
+ sleep 0.01
253
+ end
254
+ $dat[vn]
255
+ rescue =>e
256
+ p e
257
+ p $@
258
+ return nil
259
+ end
260
+
261
+ def receive_dump(sok, len)
262
+ dmp = ''
263
+ while(dmp.length != len.to_i)
264
+ dmp = dmp + sok.read(len.to_i - dmp.length)
265
+ end
266
+ sok.read(2)
267
+ if sok.gets == "END\r\n"
268
+ return Marshal.load(dmp)
269
+ else
270
+ return nil
271
+ end
272
+ rescue =>e
273
+ @log.error("#{e}\n#{$@}")
274
+ false
275
+ end
276
+
277
+ end