higgs 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. data/ChangeLog +208 -0
  2. data/LICENSE +26 -0
  3. data/README +2 -0
  4. data/Rakefile +75 -0
  5. data/bin/higgs_backup +67 -0
  6. data/bin/higgs_dump_index +43 -0
  7. data/bin/higgs_dump_jlog +42 -0
  8. data/bin/higgs_verify +37 -0
  9. data/lib/cgi/session/higgs.rb +72 -0
  10. data/lib/higgs/block.rb +192 -0
  11. data/lib/higgs/cache.rb +117 -0
  12. data/lib/higgs/dbm.rb +55 -0
  13. data/lib/higgs/exceptions.rb +31 -0
  14. data/lib/higgs/flock.rb +77 -0
  15. data/lib/higgs/index.rb +164 -0
  16. data/lib/higgs/jlog.rb +159 -0
  17. data/lib/higgs/lock.rb +189 -0
  18. data/lib/higgs/storage.rb +1086 -0
  19. data/lib/higgs/store.rb +228 -0
  20. data/lib/higgs/tar.rb +390 -0
  21. data/lib/higgs/thread.rb +370 -0
  22. data/lib/higgs/tman.rb +513 -0
  23. data/lib/higgs/utils/bman.rb +285 -0
  24. data/lib/higgs/utils.rb +22 -0
  25. data/lib/higgs/version.rb +21 -0
  26. data/lib/higgs.rb +59 -0
  27. data/misc/cache_bench/cache_bench.rb +43 -0
  28. data/misc/dbm_bench/.strc +8 -0
  29. data/misc/dbm_bench/Rakefile +78 -0
  30. data/misc/dbm_bench/dbm_multi_thread.rb +199 -0
  31. data/misc/dbm_bench/dbm_rnd_delete.rb +43 -0
  32. data/misc/dbm_bench/dbm_rnd_read.rb +44 -0
  33. data/misc/dbm_bench/dbm_rnd_update.rb +44 -0
  34. data/misc/dbm_bench/dbm_seq_read.rb +45 -0
  35. data/misc/dbm_bench/dbm_seq_write.rb +44 -0
  36. data/misc/dbm_bench/st_verify.rb +28 -0
  37. data/misc/io_bench/cksum_bench.rb +48 -0
  38. data/misc/io_bench/jlog_bench.rb +71 -0
  39. data/misc/io_bench/write_bench.rb +128 -0
  40. data/misc/thread_bench/lock_bench.rb +132 -0
  41. data/mkrdoc.rb +8 -0
  42. data/rdoc.yml +13 -0
  43. data/sample/count.rb +60 -0
  44. data/sample/dbmtest.rb +38 -0
  45. data/test/Rakefile +45 -0
  46. data/test/run.rb +32 -0
  47. data/test/test_block.rb +163 -0
  48. data/test/test_cache.rb +214 -0
  49. data/test/test_cgi_session.rb +142 -0
  50. data/test/test_flock.rb +162 -0
  51. data/test/test_index.rb +258 -0
  52. data/test/test_jlog.rb +180 -0
  53. data/test/test_lock.rb +320 -0
  54. data/test/test_online_backup.rb +169 -0
  55. data/test/test_storage.rb +439 -0
  56. data/test/test_storage_conf.rb +202 -0
  57. data/test/test_storage_init_opts.rb +89 -0
  58. data/test/test_store.rb +211 -0
  59. data/test/test_tar.rb +432 -0
  60. data/test/test_thread.rb +541 -0
  61. data/test/test_tman.rb +875 -0
  62. data/test/test_tman_init_opts.rb +56 -0
  63. data/test/test_utils_bman.rb +234 -0
  64. metadata +115 -0
@@ -0,0 +1,142 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'cgi/session/higgs'
4
+ require 'cgi/session/pstore'
5
+ require 'fileutils'
6
+ require 'logger'
7
+ require 'test/unit'
8
+
9
+ module Higgs::Test
10
+ module CGISessionTest
11
+ # for ident(1)
12
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
13
+
14
+ def new_store(session, options={})
15
+ options.update('tmpdir' => @tmpdir, :logger => @logger)
16
+ store_type.new(session, options)
17
+ end
18
+
19
+ def setup
20
+ @tmpdir = 'cgi_tmp'
21
+ FileUtils.rm_rf(@tmpdir) if $DEBUG
22
+ FileUtils.mkdir_p(@tmpdir)
23
+ @logger = proc{|path|
24
+ logger = Logger.new(path, 1)
25
+ logger.level = Logger::WARN
26
+ logger
27
+ }
28
+ @session = Object.new
29
+ class << @session
30
+ attr_accessor :session_id
31
+ attr_accessor :new_session
32
+ end
33
+ @session.session_id = 'foo'
34
+ @session.new_session = true
35
+ @store = new_store(@session)
36
+ end
37
+
38
+ def teardown
39
+ @store.close
40
+ FileUtils.rm_rf(@tmpdir) unless $DEBUG
41
+ end
42
+
43
+ def test_restore_update_close
44
+ hash = @store.restore
45
+ hash['key'] = { 'k' => 'v' }
46
+ @store.update
47
+ @store.close
48
+
49
+ @store = new_store(@session)
50
+ assert_equal({ 'key' => { 'k' => 'v' } }, @store.restore)
51
+ end
52
+
53
+ def test_delete
54
+ hash = @store.restore
55
+ hash['key'] = { 'k' => 'v' }
56
+ @store.update
57
+ @store.delete
58
+
59
+ @store = new_store(@session)
60
+ assert_equal({}, @store.restore)
61
+ end
62
+
63
+ def test_not_new_session
64
+ @store.delete
65
+
66
+ @session.new_session = false
67
+ assert_raise(CGI::Session::NoSession) {
68
+ @store = new_store(@session)
69
+ }
70
+ end
71
+
72
+ def test_multiproc_counter
73
+ @store.close
74
+ num_of_procs = 2
75
+ count_by_proc = 1000
76
+ pid_list = []
77
+ ready_latch = File.join(@tmpdir, '.ready_latch')
78
+ start_latch = File.join(@tmpdir, '.start_latch')
79
+
80
+ begin
81
+ num_of_procs.times do |i|
82
+ pid_list << fork{
83
+ FileUtils.touch("#{ready_latch}.#{i}")
84
+ until (File.exist? start_latch)
85
+ # spin lock
86
+ end
87
+ count_by_proc.times do
88
+ @store = new_store(@session)
89
+ hash = @store.restore
90
+ hash['count'] = (hash['count'] || 0).succ
91
+ @store.close
92
+ end
93
+ }
94
+ end
95
+
96
+ num_of_procs.times do |i|
97
+ until (File.exist? "#{ready_latch}.#{i}")
98
+ # spin lock
99
+ end
100
+ end
101
+ FileUtils.touch(start_latch)
102
+
103
+ for pid in pid_list
104
+ Process.waitpid(pid)
105
+ end
106
+
107
+ @store = new_store(@session)
108
+ assert_equal(num_of_procs * count_by_proc, @store.restore['count'])
109
+ ensure
110
+ num_of_procs.times do |i|
111
+ FileUtils.rm_f("#{ready_latch}.#{i}")
112
+ end
113
+ FileUtils.rm_f(start_latch)
114
+ end
115
+ end
116
+ end
117
+
118
+ class CGISessionPstoreTest < Test::Unit::TestCase
119
+ include CGISessionTest
120
+
121
+ # for ident(1)
122
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
123
+
124
+ def store_type
125
+ CGI::Session::PStore
126
+ end
127
+
128
+ # race condition
129
+ undef test_multiproc_counter
130
+ end
131
+
132
+ class CGISessionHiggsStoreTest < Test::Unit::TestCase
133
+ include CGISessionTest
134
+
135
+ # for ident(1)
136
+ CVS_ID = '$Id: test_cgi_session.rb 560 2007-09-25 15:34:24Z toki $'
137
+
138
+ def store_type
139
+ CGI::Session::HiggsStore
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,162 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'fileutils'
4
+ require 'higgs/flock'
5
+ require 'test/unit'
6
+
7
+ module Higgs::Test
8
+ class FileLockTest < Test::Unit::TestCase
9
+ include Higgs
10
+
11
+ # for ident(1)
12
+ CVS_ID = '$Id: test_flock.rb 559 2007-09-25 15:20:20Z toki $'
13
+
14
+ def setup
15
+ @test_dir = 'flock_test'
16
+ FileUtils.rm_rf(@test_dir) # for debug
17
+ FileUtils.mkdir_p(@test_dir)
18
+ @count_path = File.join(@test_dir, 'count')
19
+ @lock_path = File.join(@test_dir, 'lock')
20
+ end
21
+
22
+ def teardown
23
+ FileUtils.rm_rf(@test_dir) unless $DEBUG
24
+ end
25
+
26
+ def make_flock(*args)
27
+ flock = FileLock.new(@lock_path, *args)
28
+ begin
29
+ yield(flock)
30
+ ensure
31
+ flock.close
32
+ end
33
+ end
34
+ private :make_flock
35
+
36
+ def test_write_lock_single_process
37
+ make_flock{|flock|
38
+ File.open(@count_path, 'w') {|f| f << '0' }
39
+ 100.times do
40
+ flock.synchronize(:EX) {
41
+ value = IO.read(@count_path)
42
+ value.succ!
43
+ File.open(@count_path, 'w') {|f| f << value }
44
+ }
45
+ end
46
+ assert_equal('100', IO.read(@count_path))
47
+ }
48
+ end
49
+
50
+ def test_write_lock_multi_process
51
+ File.open(@count_path, 'w') {|f| f << '0' }
52
+
53
+ writers = 10
54
+ each_count = 1000
55
+ w_pid_list = []
56
+
57
+ writers.times do |i|
58
+ w_pid_list << fork{
59
+ make_flock{|flock|
60
+ each_count.times do
61
+ flock.synchronize(:EX) {
62
+ value = IO.read(@count_path)
63
+ value.succ!
64
+ File.open(@count_path, 'w') {|f| f << value }
65
+ }
66
+ end
67
+ }
68
+ }
69
+ end
70
+
71
+ w_st_list = []
72
+ for w_pid in w_pid_list
73
+ Process.waitpid(w_pid)
74
+ w_st_list << $?.exitstatus
75
+ end
76
+
77
+ w_st_list.each_with_index do |exitstatus, i|
78
+ assert_equal(0, exitstatus, "writer process: #{i}")
79
+ end
80
+
81
+ assert_equal(writers * each_count, IO.read(@count_path).to_i)
82
+ end
83
+
84
+ def test_read_write_lock_multi_process
85
+ File.open(@count_path, 'w') {|f| f << '0' }
86
+
87
+ writers = 3
88
+ readers = 10
89
+ each_count = 1000
90
+ w_pid_list = []
91
+ r_pid_list = []
92
+
93
+ writers.times do |i|
94
+ w_pid_list << fork{
95
+ make_flock{|flock|
96
+ each_count.times do
97
+ flock.synchronize(:EX) {
98
+ File.open(@count_path, 'w') {|f| f << '1' }
99
+ File.open(@count_path, 'w') {|f| f << '0' }
100
+ }
101
+ end
102
+ }
103
+ }
104
+ end
105
+
106
+ readers.times do |i|
107
+ r_pid_list << fork{
108
+ make_flock(true) {|flock|
109
+ each_count.times do
110
+ flock.synchronize(:SH) {
111
+ assert_equal('0', IO.read(@count_path))
112
+ }
113
+ end
114
+ }
115
+ }
116
+ end
117
+
118
+ w_st_list = []
119
+ for w_pid in w_pid_list
120
+ Process.waitpid(w_pid)
121
+ w_st_list << $?.exitstatus
122
+ end
123
+
124
+ r_st_list = []
125
+ for r_pid in r_pid_list
126
+ Process.waitpid(r_pid)
127
+ r_st_list << $?.exitstatus
128
+ end
129
+
130
+ w_st_list.each_with_index do |exitstatus, i|
131
+ assert_equal(0, exitstatus, "writer process: #{i}")
132
+ end
133
+
134
+ r_st_list.each_with_index do |exitstatus, i|
135
+ assert_equal(0, exitstatus, "reader process: #{i}")
136
+ end
137
+ end
138
+
139
+ def test_read_only_lock_failed_to_write_lock
140
+ make_flock(true) {|flock|
141
+ assert_raise(RuntimeError) {
142
+ flock.write_lock
143
+ }
144
+ }
145
+ end
146
+
147
+ def test_synchronize_unknown_mode_error
148
+ make_flock(true) {|flock|
149
+ assert_raise(ArgumentError) {
150
+ flock.synchronize(:UNKNOWN) {
151
+ flunk('not to reach')
152
+ }
153
+ }
154
+ }
155
+ end
156
+ end
157
+ end
158
+
159
+ # Local Variables:
160
+ # mode: Ruby
161
+ # indent-tabs-mode: nil
162
+ # End:
@@ -0,0 +1,258 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'fileutils'
4
+ require 'higgs/block'
5
+ require 'higgs/index'
6
+ require 'test/unit'
7
+
8
+ module Higgs::Test
9
+ class IndexChangeNumberTest < Test::Unit::TestCase
10
+ include Higgs
11
+
12
+ # for ident(1)
13
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
14
+
15
+ def setup
16
+ @idx = Index.new
17
+ end
18
+
19
+ def test_change_number
20
+ assert_equal(0, @idx.change_number)
21
+ @idx.succ!
22
+ assert_equal(1, @idx.change_number)
23
+ @idx.succ!
24
+ assert_equal(2, @idx.change_number)
25
+ @idx.succ!
26
+ assert_equal(3, @idx.change_number)
27
+ @idx.succ!
28
+ assert_equal(4, @idx.change_number)
29
+ @idx.succ!
30
+ assert_equal(5, @idx.change_number)
31
+ @idx.succ!
32
+ assert_equal(6, @idx.change_number)
33
+ @idx.succ!
34
+ assert_equal(7, @idx.change_number)
35
+ @idx.succ!
36
+ assert_equal(8, @idx.change_number)
37
+ @idx.succ!
38
+ assert_equal(9, @idx.change_number)
39
+ @idx.succ!
40
+ assert_equal(10, @idx.change_number)
41
+ end
42
+ end
43
+
44
+ class IndexFreeListTest < Test::Unit::TestCase
45
+ include Higgs
46
+
47
+ # for ident(1)
48
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
49
+
50
+ def setup
51
+ @idx = Index.new
52
+ end
53
+
54
+ def test_free_single
55
+ assert_equal(nil, @idx.free_fetch(512))
56
+ @idx.free_store(0, 512)
57
+ assert_equal(0, @idx.free_fetch(512))
58
+ assert_equal(nil, @idx.free_fetch(512))
59
+ end
60
+
61
+ def test_free_multi_size
62
+ @idx.free_store(0, 1024)
63
+ @idx.free_store(3584, 8192)
64
+ assert_equal(nil, @idx.free_fetch(512))
65
+ assert_equal(0, @idx.free_fetch(1024))
66
+ assert_equal(3584, @idx.free_fetch(8192))
67
+ end
68
+
69
+ def test_free_multi_segment
70
+ @idx.free_store(0, 512)
71
+ @idx.free_store(8192, 512)
72
+ @idx.free_store(9782, 512)
73
+ assert_equal(0, @idx.free_fetch(512))
74
+ assert_equal(8192, @idx.free_fetch(512))
75
+ assert_equal(9782, @idx.free_fetch(512))
76
+ assert_equal(nil, @idx.free_fetch(512))
77
+ end
78
+
79
+ def test_free_fetch_at
80
+ @idx.free_store(0, 512)
81
+ @idx.free_store(8192, 512)
82
+ @idx.free_store(9782, 512)
83
+ assert_equal(0, @idx.free_fetch_at(0, 512))
84
+ assert_equal(8192, @idx.free_fetch_at(8192, 512))
85
+ assert_equal(9782, @idx.free_fetch_at(9782, 512))
86
+ assert_equal(nil, @idx.free_fetch(512))
87
+ end
88
+ end
89
+
90
+ class IndexTest < Test::Unit::TestCase
91
+ include Higgs
92
+
93
+ # for ident(1)
94
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
95
+
96
+ def setup
97
+ @idx = Index.new
98
+ end
99
+
100
+ def test_single_entry
101
+ assert_equal(nil, @idx['foo'])
102
+ @idx['foo'] = 0
103
+ assert_equal(0, @idx['foo'])
104
+ end
105
+
106
+ def test_multi_entry
107
+ assert_equal(nil, @idx['foo'])
108
+ assert_equal(nil, @idx['bar'])
109
+ @idx['foo'] = 0
110
+ @idx['bar'] = 8192
111
+ assert_equal(0, @idx['foo'])
112
+ assert_equal(8192, @idx['bar'])
113
+ end
114
+
115
+ def test_key
116
+ assert_equal(false, (@idx.key? 'foo'))
117
+ @idx['foo'] = 0
118
+ assert_equal(true, (@idx.key? 'foo'))
119
+ end
120
+
121
+ def test_keys
122
+ assert_equal([], @idx.keys)
123
+ @idx['foo'] = 0
124
+ assert_equal(%w[ foo ].sort, @idx.keys.sort)
125
+ @idx['bar'] = 512
126
+ assert_equal(%w[ foo bar ].sort, @idx.keys.sort)
127
+ @idx['baz'] = 1024
128
+ assert_equal(%w[ foo bar baz ].sort, @idx.keys.sort)
129
+ end
130
+
131
+ def test_each_key
132
+ assert_equal(@idx, @idx.each_key{|key| assert_fail('not to reach') })
133
+
134
+ @idx['foo'] = 0
135
+ expected_keys = %w[ foo ]
136
+ @idx.each_key do |key|
137
+ assert(expected_keys.delete(key), "key: #{key}")
138
+ end
139
+ assert_equal([], expected_keys)
140
+
141
+ @idx['bar'] = 512
142
+ expected_keys = %w[ foo bar ]
143
+ @idx.each_key do |key|
144
+ assert(expected_keys.delete(key), "key: #{key}")
145
+ end
146
+ assert_equal([], expected_keys)
147
+
148
+ @idx['baz'] = 1024
149
+ expected_keys = %w[ foo bar baz ]
150
+ @idx.each_key do |key|
151
+ assert(expected_keys.delete(key), "key: #{key}")
152
+ end
153
+ assert_equal([], expected_keys)
154
+ end
155
+
156
+ def test_delete
157
+ assert_equal(nil, @idx.delete('foo'))
158
+ @idx['foo'] = 0
159
+ assert_equal(0, @idx.delete('foo'))
160
+ end
161
+ end
162
+
163
+ class IndexIdentitiesTest < Test::Unit::TestCase
164
+ include Higgs
165
+
166
+ # for ident(1)
167
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
168
+
169
+ def setup
170
+ @idx = Index.new
171
+ end
172
+
173
+ def test_identity
174
+ @idx[:foo] = 0
175
+ assert_equal('foo', @idx.identity(:foo))
176
+ end
177
+
178
+ def test_identity_not_defined
179
+ assert_nil(@idx.identity(:foo))
180
+ end
181
+
182
+ def test_identity_dup
183
+ @idx[:foo] = 0
184
+ @idx['foo'] = 1
185
+ @idx[ %w[ f o o ] ] = 2
186
+ assert_equal('foo', @idx.identity(:foo))
187
+ assert_equal('foo.a', @idx.identity('foo'))
188
+ assert_equal('foo.b', @idx.identity(%w[ f o o ]))
189
+ end
190
+
191
+ def test_delete
192
+ @idx[:foo] = 0
193
+ @idx['foo'] = 1
194
+ @idx.delete(:foo)
195
+ assert_equal(nil, @idx.identity(:foo))
196
+ assert_equal('foo.a', @idx.identity('foo'))
197
+ end
198
+ end
199
+
200
+ class IndexLoadSaveTest < Test::Unit::TestCase
201
+ include Higgs
202
+
203
+ # for ident(1)
204
+ CVS_ID = '$Id: test_index.rb 559 2007-09-25 15:20:20Z toki $'
205
+
206
+ def setup
207
+ @path = 'test.idx'
208
+ end
209
+
210
+ def teardown
211
+ FileUtils.rm_f(@path)
212
+ end
213
+
214
+ def test_save_load
215
+ i = Index.new
216
+ i.succ!
217
+ i.free_store(0, 512)
218
+ i[:foo] = 1024
219
+ i.eoa = 2048
220
+ i.save(@path)
221
+
222
+ j = Index.new
223
+ j.load(@path)
224
+ assert_equal(1, j.change_number)
225
+ assert_equal(2048, j.eoa)
226
+ assert_equal(0, j.free_fetch(512))
227
+ assert_equal(1024, j[:foo])
228
+ assert_equal('foo', j.identity(:foo))
229
+ end
230
+
231
+ def test_migration
232
+ index_data_0_0 = {
233
+ :version => [ 0, 0 ],
234
+ :change_number => 0,
235
+ :eoa => 1024,
236
+ :free_lists => { 512 => [ 512 ] },
237
+ :index => { :foo => 0 }
238
+ }
239
+ File.open(@path, 'w') {|w|
240
+ w.binmode
241
+ Block.block_write(w, Index::MAGIC_SYMBOL, Marshal.dump(index_data_0_0))
242
+ }
243
+
244
+ i = Index.new
245
+ i.load(@path)
246
+ assert_equal(0, i.change_number)
247
+ assert_equal(1024, i.eoa)
248
+ assert_equal(512, i.free_fetch(512))
249
+ assert_equal(0, i[:foo])
250
+ assert_equal('foo', i.identity(:foo))
251
+ end
252
+ end
253
+ end
254
+
255
+ # Local Variables:
256
+ # mode: Ruby
257
+ # indent-tabs-mode: nil
258
+ # End: