higgs 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (64) hide show
  1. data/ChangeLog +208 -0
  2. data/LICENSE +26 -0
  3. data/README +2 -0
  4. data/Rakefile +75 -0
  5. data/bin/higgs_backup +67 -0
  6. data/bin/higgs_dump_index +43 -0
  7. data/bin/higgs_dump_jlog +42 -0
  8. data/bin/higgs_verify +37 -0
  9. data/lib/cgi/session/higgs.rb +72 -0
  10. data/lib/higgs/block.rb +192 -0
  11. data/lib/higgs/cache.rb +117 -0
  12. data/lib/higgs/dbm.rb +55 -0
  13. data/lib/higgs/exceptions.rb +31 -0
  14. data/lib/higgs/flock.rb +77 -0
  15. data/lib/higgs/index.rb +164 -0
  16. data/lib/higgs/jlog.rb +159 -0
  17. data/lib/higgs/lock.rb +189 -0
  18. data/lib/higgs/storage.rb +1086 -0
  19. data/lib/higgs/store.rb +228 -0
  20. data/lib/higgs/tar.rb +390 -0
  21. data/lib/higgs/thread.rb +370 -0
  22. data/lib/higgs/tman.rb +513 -0
  23. data/lib/higgs/utils/bman.rb +285 -0
  24. data/lib/higgs/utils.rb +22 -0
  25. data/lib/higgs/version.rb +21 -0
  26. data/lib/higgs.rb +59 -0
  27. data/misc/cache_bench/cache_bench.rb +43 -0
  28. data/misc/dbm_bench/.strc +8 -0
  29. data/misc/dbm_bench/Rakefile +78 -0
  30. data/misc/dbm_bench/dbm_multi_thread.rb +199 -0
  31. data/misc/dbm_bench/dbm_rnd_delete.rb +43 -0
  32. data/misc/dbm_bench/dbm_rnd_read.rb +44 -0
  33. data/misc/dbm_bench/dbm_rnd_update.rb +44 -0
  34. data/misc/dbm_bench/dbm_seq_read.rb +45 -0
  35. data/misc/dbm_bench/dbm_seq_write.rb +44 -0
  36. data/misc/dbm_bench/st_verify.rb +28 -0
  37. data/misc/io_bench/cksum_bench.rb +48 -0
  38. data/misc/io_bench/jlog_bench.rb +71 -0
  39. data/misc/io_bench/write_bench.rb +128 -0
  40. data/misc/thread_bench/lock_bench.rb +132 -0
  41. data/mkrdoc.rb +8 -0
  42. data/rdoc.yml +13 -0
  43. data/sample/count.rb +60 -0
  44. data/sample/dbmtest.rb +38 -0
  45. data/test/Rakefile +45 -0
  46. data/test/run.rb +32 -0
  47. data/test/test_block.rb +163 -0
  48. data/test/test_cache.rb +214 -0
  49. data/test/test_cgi_session.rb +142 -0
  50. data/test/test_flock.rb +162 -0
  51. data/test/test_index.rb +258 -0
  52. data/test/test_jlog.rb +180 -0
  53. data/test/test_lock.rb +320 -0
  54. data/test/test_online_backup.rb +169 -0
  55. data/test/test_storage.rb +439 -0
  56. data/test/test_storage_conf.rb +202 -0
  57. data/test/test_storage_init_opts.rb +89 -0
  58. data/test/test_store.rb +211 -0
  59. data/test/test_tar.rb +432 -0
  60. data/test/test_thread.rb +541 -0
  61. data/test/test_tman.rb +875 -0
  62. data/test/test_tman_init_opts.rb +56 -0
  63. data/test/test_utils_bman.rb +234 -0
  64. metadata +115 -0
@@ -0,0 +1,142 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'cgi/session/higgs'
4
+ require 'cgi/session/pstore'
5
+ require 'fileutils'
6
+ require 'logger'
7
+ require 'test/unit'
8
+
9
+ module Higgs::Test
10
+ module CGISessionTest
11
+ # for ident(1)
12
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
13
+
14
+ def new_store(session, options={})
15
+ options.update('tmpdir' => @tmpdir, :logger => @logger)
16
+ store_type.new(session, options)
17
+ end
18
+
19
+ def setup
20
+ @tmpdir = 'cgi_tmp'
21
+ FileUtils.rm_rf(@tmpdir) if $DEBUG
22
+ FileUtils.mkdir_p(@tmpdir)
23
+ @logger = proc{|path|
24
+ logger = Logger.new(path, 1)
25
+ logger.level = Logger::WARN
26
+ logger
27
+ }
28
+ @session = Object.new
29
+ class << @session
30
+ attr_accessor :session_id
31
+ attr_accessor :new_session
32
+ end
33
+ @session.session_id = 'foo'
34
+ @session.new_session = true
35
+ @store = new_store(@session)
36
+ end
37
+
38
+ def teardown
39
+ @store.close
40
+ FileUtils.rm_rf(@tmpdir) unless $DEBUG
41
+ end
42
+
43
+ def test_restore_update_close
44
+ hash = @store.restore
45
+ hash['key'] = { 'k' => 'v' }
46
+ @store.update
47
+ @store.close
48
+
49
+ @store = new_store(@session)
50
+ assert_equal({ 'key' => { 'k' => 'v' } }, @store.restore)
51
+ end
52
+
53
+ def test_delete
54
+ hash = @store.restore
55
+ hash['key'] = { 'k' => 'v' }
56
+ @store.update
57
+ @store.delete
58
+
59
+ @store = new_store(@session)
60
+ assert_equal({}, @store.restore)
61
+ end
62
+
63
+ def test_not_new_session
64
+ @store.delete
65
+
66
+ @session.new_session = false
67
+ assert_raise(CGI::Session::NoSession) {
68
+ @store = new_store(@session)
69
+ }
70
+ end
71
+
72
+ def test_multiproc_counter
73
+ @store.close
74
+ num_of_procs = 2
75
+ count_by_proc = 1000
76
+ pid_list = []
77
+ ready_latch = File.join(@tmpdir, '.ready_latch')
78
+ start_latch = File.join(@tmpdir, '.start_latch')
79
+
80
+ begin
81
+ num_of_procs.times do |i|
82
+ pid_list << fork{
83
+ FileUtils.touch("#{ready_latch}.#{i}")
84
+ until (File.exist? start_latch)
85
+ # spin lock
86
+ end
87
+ count_by_proc.times do
88
+ @store = new_store(@session)
89
+ hash = @store.restore
90
+ hash['count'] = (hash['count'] || 0).succ
91
+ @store.close
92
+ end
93
+ }
94
+ end
95
+
96
+ num_of_procs.times do |i|
97
+ until (File.exist? "#{ready_latch}.#{i}")
98
+ # spin lock
99
+ end
100
+ end
101
+ FileUtils.touch(start_latch)
102
+
103
+ for pid in pid_list
104
+ Process.waitpid(pid)
105
+ end
106
+
107
+ @store = new_store(@session)
108
+ assert_equal(num_of_procs * count_by_proc, @store.restore['count'])
109
+ ensure
110
+ num_of_procs.times do |i|
111
+ FileUtils.rm_f("#{ready_latch}.#{i}")
112
+ end
113
+ FileUtils.rm_f(start_latch)
114
+ end
115
+ end
116
+ end
117
+
118
+ class CGISessionPstoreTest < Test::Unit::TestCase
119
+ include CGISessionTest
120
+
121
+ # for ident(1)
122
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
123
+
124
+ def store_type
125
+ CGI::Session::PStore
126
+ end
127
+
128
+ # race condition
129
+ undef test_multiproc_counter
130
+ end
131
+
132
+ class CGISessionHiggsStoreTest < Test::Unit::TestCase
133
+ include CGISessionTest
134
+
135
+ # for ident(1)
136
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
137
+
138
+ def store_type
139
+ CGI::Session::HiggsStore
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,162 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'fileutils'
4
+ require 'higgs/flock'
5
+ require 'test/unit'
6
+
7
+ module Higgs::Test
8
+ class FileLockTest < Test::Unit::TestCase
9
+ include Higgs
10
+
11
+ # for ident(1)
12
+ CVS_ID = '$Id: test_flock.rb 559 2007-09-25 15:20:20Z toki $'
13
+
14
+ def setup
15
+ @test_dir = 'flock_test'
16
+ FileUtils.rm_rf(@test_dir) # for debug
17
+ FileUtils.mkdir_p(@test_dir)
18
+ @count_path = File.join(@test_dir, 'count')
19
+ @lock_path = File.join(@test_dir, 'lock')
20
+ end
21
+
22
+ def teardown
23
+ FileUtils.rm_rf(@test_dir) unless $DEBUG
24
+ end
25
+
26
+ def make_flock(*args)
27
+ flock = FileLock.new(@lock_path, *args)
28
+ begin
29
+ yield(flock)
30
+ ensure
31
+ flock.close
32
+ end
33
+ end
34
+ private :make_flock
35
+
36
+ def test_write_lock_single_process
37
+ make_flock{|flock|
38
+ File.open(@count_path, 'w') {|f| f << '0' }
39
+ 100.times do
40
+ flock.synchronize(:EX) {
41
+ value = IO.read(@count_path)
42
+ value.succ!
43
+ File.open(@count_path, 'w') {|f| f << value }
44
+ }
45
+ end
46
+ assert_equal('100', IO.read(@count_path))
47
+ }
48
+ end
49
+
50
+ def test_write_lock_multi_process
51
+ File.open(@count_path, 'w') {|f| f << '0' }
52
+
53
+ writers = 10
54
+ each_count = 1000
55
+ w_pid_list = []
56
+
57
+ writers.times do |i|
58
+ w_pid_list << fork{
59
+ make_flock{|flock|
60
+ each_count.times do
61
+ flock.synchronize(:EX) {
62
+ value = IO.read(@count_path)
63
+ value.succ!
64
+ File.open(@count_path, 'w') {|f| f << value }
65
+ }
66
+ end
67
+ }
68
+ }
69
+ end
70
+
71
+ w_st_list = []
72
+ for w_pid in w_pid_list
73
+ Process.waitpid(w_pid)
74
+ w_st_list << $?.exitstatus
75
+ end
76
+
77
+ w_st_list.each_with_index do |exitstatus, i|
78
+ assert_equal(0, exitstatus, "writer process: #{i}")
79
+ end
80
+
81
+ assert_equal(writers * each_count, IO.read(@count_path).to_i)
82
+ end
83
+
84
+ def test_read_write_lock_multi_process
85
+ File.open(@count_path, 'w') {|f| f << '0' }
86
+
87
+ writers = 3
88
+ readers = 10
89
+ each_count = 1000
90
+ w_pid_list = []
91
+ r_pid_list = []
92
+
93
+ writers.times do |i|
94
+ w_pid_list << fork{
95
+ make_flock{|flock|
96
+ each_count.times do
97
+ flock.synchronize(:EX) {
98
+ File.open(@count_path, 'w') {|f| f << '1' }
99
+ File.open(@count_path, 'w') {|f| f << '0' }
100
+ }
101
+ end
102
+ }
103
+ }
104
+ end
105
+
106
+ readers.times do |i|
107
+ r_pid_list << fork{
108
+ make_flock(true) {|flock|
109
+ each_count.times do
110
+ flock.synchronize(:SH) {
111
+ assert_equal('0', IO.read(@count_path))
112
+ }
113
+ end
114
+ }
115
+ }
116
+ end
117
+
118
+ w_st_list = []
119
+ for w_pid in w_pid_list
120
+ Process.waitpid(w_pid)
121
+ w_st_list << $?.exitstatus
122
+ end
123
+
124
+ r_st_list = []
125
+ for r_pid in r_pid_list
126
+ Process.waitpid(r_pid)
127
+ r_st_list << $?.exitstatus
128
+ end
129
+
130
+ w_st_list.each_with_index do |exitstatus, i|
131
+ assert_equal(0, exitstatus, "writer process: #{i}")
132
+ end
133
+
134
+ r_st_list.each_with_index do |exitstatus, i|
135
+ assert_equal(0, exitstatus, "reader process: #{i}")
136
+ end
137
+ end
138
+
139
+ def test_read_only_lock_failed_to_write_lock
140
+ make_flock(true) {|flock|
141
+ assert_raise(RuntimeError) {
142
+ flock.write_lock
143
+ }
144
+ }
145
+ end
146
+
147
+ def test_synchronize_unknown_mode_error
148
+ make_flock(true) {|flock|
149
+ assert_raise(ArgumentError) {
150
+ flock.synchronize(:UNKNOWN) {
151
+ flunk('not to reach')
152
+ }
153
+ }
154
+ }
155
+ end
156
+ end
157
+ end
158
+
159
+ # Local Variables:
160
+ # mode: Ruby
161
+ # indent-tabs-mode: nil
162
+ # End:
@@ -0,0 +1,258 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'fileutils'
4
+ require 'higgs/block'
5
+ require 'higgs/index'
6
+ require 'test/unit'
7
+
8
+ module Higgs::Test
9
+ class IndexChangeNumberTest < Test::Unit::TestCase
10
+ include Higgs
11
+
12
+ # for ident(1)
13
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
14
+
15
+ def setup
16
+ @idx = Index.new
17
+ end
18
+
19
+ def test_change_number
20
+ assert_equal(0, @idx.change_number)
21
+ @idx.succ!
22
+ assert_equal(1, @idx.change_number)
23
+ @idx.succ!
24
+ assert_equal(2, @idx.change_number)
25
+ @idx.succ!
26
+ assert_equal(3, @idx.change_number)
27
+ @idx.succ!
28
+ assert_equal(4, @idx.change_number)
29
+ @idx.succ!
30
+ assert_equal(5, @idx.change_number)
31
+ @idx.succ!
32
+ assert_equal(6, @idx.change_number)
33
+ @idx.succ!
34
+ assert_equal(7, @idx.change_number)
35
+ @idx.succ!
36
+ assert_equal(8, @idx.change_number)
37
+ @idx.succ!
38
+ assert_equal(9, @idx.change_number)
39
+ @idx.succ!
40
+ assert_equal(10, @idx.change_number)
41
+ end
42
+ end
43
+
44
+ class IndexFreeListTest < Test::Unit::TestCase
45
+ include Higgs
46
+
47
+ # for ident(1)
48
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
49
+
50
+ def setup
51
+ @idx = Index.new
52
+ end
53
+
54
+ def test_free_single
55
+ assert_equal(nil, @idx.free_fetch(512))
56
+ @idx.free_store(0, 512)
57
+ assert_equal(0, @idx.free_fetch(512))
58
+ assert_equal(nil, @idx.free_fetch(512))
59
+ end
60
+
61
+ def test_free_multi_size
62
+ @idx.free_store(0, 1024)
63
+ @idx.free_store(3584, 8192)
64
+ assert_equal(nil, @idx.free_fetch(512))
65
+ assert_equal(0, @idx.free_fetch(1024))
66
+ assert_equal(3584, @idx.free_fetch(8192))
67
+ end
68
+
69
+ def test_free_multi_segment
70
+ @idx.free_store(0, 512)
71
+ @idx.free_store(8192, 512)
72
+ @idx.free_store(9782, 512)
73
+ assert_equal(0, @idx.free_fetch(512))
74
+ assert_equal(8192, @idx.free_fetch(512))
75
+ assert_equal(9782, @idx.free_fetch(512))
76
+ assert_equal(nil, @idx.free_fetch(512))
77
+ end
78
+
79
+ def test_free_fetch_at
80
+ @idx.free_store(0, 512)
81
+ @idx.free_store(8192, 512)
82
+ @idx.free_store(9782, 512)
83
+ assert_equal(0, @idx.free_fetch_at(0, 512))
84
+ assert_equal(8192, @idx.free_fetch_at(8192, 512))
85
+ assert_equal(9782, @idx.free_fetch_at(9782, 512))
86
+ assert_equal(nil, @idx.free_fetch(512))
87
+ end
88
+ end
89
+
90
+ class IndexTest < Test::Unit::TestCase
91
+ include Higgs
92
+
93
+ # for ident(1)
94
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
95
+
96
+ def setup
97
+ @idx = Index.new
98
+ end
99
+
100
+ def test_single_entry
101
+ assert_equal(nil, @idx['foo'])
102
+ @idx['foo'] = 0
103
+ assert_equal(0, @idx['foo'])
104
+ end
105
+
106
+ def test_multi_entry
107
+ assert_equal(nil, @idx['foo'])
108
+ assert_equal(nil, @idx['bar'])
109
+ @idx['foo'] = 0
110
+ @idx['bar'] = 8192
111
+ assert_equal(0, @idx['foo'])
112
+ assert_equal(8192, @idx['bar'])
113
+ end
114
+
115
+ def test_key
116
+ assert_equal(false, (@idx.key? 'foo'))
117
+ @idx['foo'] = 0
118
+ assert_equal(true, (@idx.key? 'foo'))
119
+ end
120
+
121
+ def test_keys
122
+ assert_equal([], @idx.keys)
123
+ @idx['foo'] = 0
124
+ assert_equal(%w[ foo ].sort, @idx.keys.sort)
125
+ @idx['bar'] = 512
126
+ assert_equal(%w[ foo bar ].sort, @idx.keys.sort)
127
+ @idx['baz'] = 1024
128
+ assert_equal(%w[ foo bar baz ].sort, @idx.keys.sort)
129
+ end
130
+
131
+ def test_each_key
132
+ assert_equal(@idx, @idx.each_key{|key| assert_fail('not to reach') })
133
+
134
+ @idx['foo'] = 0
135
+ expected_keys = %w[ foo ]
136
+ @idx.each_key do |key|
137
+ assert(expected_keys.delete(key), "key: #{key}")
138
+ end
139
+ assert_equal([], expected_keys)
140
+
141
+ @idx['bar'] = 512
142
+ expected_keys = %w[ foo bar ]
143
+ @idx.each_key do |key|
144
+ assert(expected_keys.delete(key), "key: #{key}")
145
+ end
146
+ assert_equal([], expected_keys)
147
+
148
+ @idx['baz'] = 1024
149
+ expected_keys = %w[ foo bar baz ]
150
+ @idx.each_key do |key|
151
+ assert(expected_keys.delete(key), "key: #{key}")
152
+ end
153
+ assert_equal([], expected_keys)
154
+ end
155
+
156
+ def test_delete
157
+ assert_equal(nil, @idx.delete('foo'))
158
+ @idx['foo'] = 0
159
+ assert_equal(0, @idx.delete('foo'))
160
+ end
161
+ end
162
+
163
+ class IndexIdentitiesTest < Test::Unit::TestCase
164
+ include Higgs
165
+
166
+ # for ident(1)
167
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
168
+
169
+ def setup
170
+ @idx = Index.new
171
+ end
172
+
173
+ def test_identity
174
+ @idx[:foo] = 0
175
+ assert_equal('foo', @idx.identity(:foo))
176
+ end
177
+
178
+ def test_identity_not_defined
179
+ assert_nil(@idx.identity(:foo))
180
+ end
181
+
182
+ def test_identity_dup
183
+ @idx[:foo] = 0
184
+ @idx['foo'] = 1
185
+ @idx[ %w[ f o o ] ] = 2
186
+ assert_equal('foo', @idx.identity(:foo))
187
+ assert_equal('foo.a', @idx.identity('foo'))
188
+ assert_equal('foo.b', @idx.identity(%w[ f o o ]))
189
+ end
190
+
191
+ def test_delete
192
+ @idx[:foo] = 0
193
+ @idx['foo'] = 1
194
+ @idx.delete(:foo)
195
+ assert_equal(nil, @idx.identity(:foo))
196
+ assert_equal('foo.a', @idx.identity('foo'))
197
+ end
198
+ end
199
+
200
+ class IndexLoadSaveTest < Test::Unit::TestCase
201
+ include Higgs
202
+
203
+ # for ident(1)
204
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
205
+
206
+ def setup
207
+ @path = 'test.idx'
208
+ end
209
+
210
+ def teardown
211
+ FileUtils.rm_f(@path)
212
+ end
213
+
214
+ def test_save_load
215
+ i = Index.new
216
+ i.succ!
217
+ i.free_store(0, 512)
218
+ i[:foo] = 1024
219
+ i.eoa = 2048
220
+ i.save(@path)
221
+
222
+ j = Index.new
223
+ j.load(@path)
224
+ assert_equal(1, j.change_number)
225
+ assert_equal(2048, j.eoa)
226
+ assert_equal(0, j.free_fetch(512))
227
+ assert_equal(1024, j[:foo])
228
+ assert_equal('foo', j.identity(:foo))
229
+ end
230
+
231
+ def test_migration
232
+ index_data_0_0 = {
233
+ :version => [ 0, 0 ],
234
+ :change_number => 0,
235
+ :eoa => 1024,
236
+ :free_lists => { 512 => [ 512 ] },
237
+ :index => { :foo => 0 }
238
+ }
239
+ File.open(@path, 'w') {|w|
240
+ w.binmode
241
+ Block.block_write(w, Index::MAGIC_SYMBOL, Marshal.dump(index_data_0_0))
242
+ }
243
+
244
+ i = Index.new
245
+ i.load(@path)
246
+ assert_equal(0, i.change_number)
247
+ assert_equal(1024, i.eoa)
248
+ assert_equal(512, i.free_fetch(512))
249
+ assert_equal(0, i[:foo])
250
+ assert_equal('foo', i.identity(:foo))
251
+ end
252
+ end
253
+ end
254
+
255
+ # Local Variables:
256
+ # mode: Ruby
257
+ # indent-tabs-mode: nil
258
+ # End: