higgs 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  # = version
2
2
  #
3
3
  # Author:: $Author: toki $
4
- # Date:: $Date: 2007-11-28 22:51:52 +0900 (Wed, 28 Nov 2007) $
5
- # Revision:: $Revision: 689 $
4
+ # Date:: $Date: 2008-01-20 23:03:17 +0900 (Sun, 20 Jan 2008) $
5
+ # Revision:: $Revision: 747 $
6
6
  #
7
7
  # == license
8
8
  # :include:LICENSE
@@ -10,9 +10,9 @@
10
10
 
11
11
  module Higgs
12
12
  # for ident(1)
13
- CVS_ID = '$Id: version.rb 689 2007-11-28 13:51:52Z toki $'
13
+ CVS_ID = '$Id: version.rb 747 2008-01-20 14:03:17Z toki $'
14
14
 
15
- VERSION = '0.1.4'
15
+ VERSION = '0.1.5'
16
16
  end
17
17
 
18
18
  # Local Variables:
@@ -5,12 +5,14 @@ require 'higgs/thread'
5
5
  require 'test/unit'
6
6
  require 'timeout'
7
7
 
8
+ Thread.abort_on_exception = true if $DEBUG
9
+
8
10
  module Higgs::Test
9
11
  class LRUCacheTest < Test::Unit::TestCase
10
12
  include Higgs
11
13
 
12
14
  # for ident(1)
13
- CVS_ID = '$Id: test_cache.rb 659 2007-10-30 15:36:09Z toki $'
15
+ CVS_ID = '$Id: test_cache.rb 742 2008-01-13 15:55:53Z toki $'
14
16
 
15
17
  CACHE_LIMIT_SIZE = 10
16
18
 
@@ -84,7 +86,7 @@ module Higgs::Test
84
86
  include Higgs
85
87
 
86
88
  # for ident(1)
87
- CVS_ID = '$Id: test_cache.rb 659 2007-10-30 15:36:09Z toki $'
89
+ CVS_ID = '$Id: test_cache.rb 742 2008-01-13 15:55:53Z toki $'
88
90
 
89
91
  def setup
90
92
  @calc_calls = 0
@@ -212,7 +214,7 @@ module Higgs::Test
212
214
  include Higgs
213
215
 
214
216
  # for ident(1)
215
- CVS_ID = '$Id: test_cache.rb 659 2007-10-30 15:36:09Z toki $'
217
+ CVS_ID = '$Id: test_cache.rb 742 2008-01-13 15:55:53Z toki $'
216
218
 
217
219
  def test_no_work_block
218
220
  assert_raise(ArgumentError) {
@@ -0,0 +1,444 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'fileutils'
4
+ require 'higgs/cache'
5
+ require 'higgs/storage'
6
+ require 'higgs/thread'
7
+ require 'logger'
8
+ require 'test/unit'
9
+
10
+ Thread.abort_on_exception = true if $DEBUG
11
+
12
+ module Higgs::Test
13
+ class MVCCCacheTest < Test::Unit::TestCase
14
+ include Higgs
15
+
16
+ # for ident(1)
17
+ CVS_ID = '$Id: test_cache_mvcc.rb 742 2008-01-13 15:55:53Z toki $'
18
+
19
+ NUM_OF_THREADS = 10
20
+ WORK_COUNT = 10000
21
+
22
+ def setup
23
+ srand(0) # prset for rand
24
+ @cache = MVCCCache.new
25
+ end
26
+
27
+ def test_many_transaction
28
+ assert(@cache.empty?)
29
+
30
+ cnum_list = [ 2, 7, 13 ]
31
+ barrier = Higgs::Barrier.new(NUM_OF_THREADS + 1)
32
+ th_grp = ThreadGroup.new
33
+
34
+ NUM_OF_THREADS.times{|i|
35
+ th_grp.add Thread.new{
36
+ barrier.wait
37
+ WORK_COUNT.times do
38
+ cnum = cnum_list[rand(cnum_list.size)]
39
+ @cache.transaction(cnum) {
40
+ assert(! @cache.empty?, "#thread: {i}th")
41
+ }
42
+ end
43
+ }
44
+ }
45
+
46
+ barrier.wait
47
+ for t in th_grp.list
48
+ t.join
49
+ end
50
+
51
+ assert(@cache.empty?)
52
+ end
53
+
54
+ MVCC_WARMUP_COUNT = 10
55
+
56
+ class RunFlag
57
+ def initialize(running)
58
+ #@lock = Mutex.new
59
+ @running = running
60
+ end
61
+
62
+ def running=(running)
63
+ #@lock.synchronize{
64
+ @running = running
65
+ #}
66
+ end
67
+
68
+ def running?
69
+ #@lock.synchronize{
70
+ @running
71
+ #}
72
+ end
73
+ end
74
+
75
+ class Count
76
+ def initialize
77
+ @value = 0
78
+ @lock = Mutex.new
79
+ end
80
+
81
+ def succ!
82
+ @lock.synchronize{ @value += 1 }
83
+ end
84
+
85
+ def value
86
+ @lock.synchronize{ @value }
87
+ end
88
+ end
89
+
90
+ def test_mvcc
91
+ store = {}
92
+ do_read = RunFlag.new(true)
93
+
94
+ # change number: 0
95
+ init_read_latch = Latch.new
96
+ init_read_count = Count.new
97
+ init_read_thread = Thread.new{
98
+ @cache.transaction(0) {|snapshot|
99
+ while (do_read.running?)
100
+ p [ 0, snapshot ] if $DEBUG
101
+ assert_equal([], snapshot.keys(store))
102
+ assert_equal(false, (snapshot.key? store, :foo))
103
+ assert_nil(snapshot.fetch(:foo, :d) { store[:foo] })
104
+ init_read_count.succ!
105
+ init_read_latch.start if (init_read_count.value == MVCC_WARMUP_COUNT)
106
+ end
107
+ }
108
+ }
109
+ init_read_latch.wait
110
+
111
+ # insert
112
+ @cache.write_old_values(0, [ [ :none, :foo ] ])
113
+ store[:foo] = 'Hello world.'
114
+
115
+ # change number: 1
116
+ insert_read_latch = Latch.new
117
+ insert_read_count = Count.new
118
+ insert_read_thread = Thread.new{
119
+ @cache.transaction(1) {|snapshot|
120
+ while (do_read.running?)
121
+ p [ 1, snapshot ] if $DEBUG
122
+ assert_equal([ :foo ], snapshot.keys(store))
123
+ assert_equal(true, (snapshot.key? store, :foo))
124
+ assert_equal('Hello world.', snapshot.fetch(:foo, :d) { store[:foo] })
125
+ insert_read_count.succ!
126
+ insert_read_latch.start if (insert_read_count.value == MVCC_WARMUP_COUNT)
127
+ end
128
+ }
129
+ }
130
+ insert_read_latch.wait
131
+
132
+ # update
133
+ @cache.write_old_values(1, [ [ :value, :foo, :d, store[:foo] ] ])
134
+ store[:foo] = 'I like ruby.'
135
+
136
+ # change number: 2
137
+ update_read_latch = Latch.new
138
+ update_read_count = Count.new
139
+ update_read_thread = Thread.new{
140
+ @cache.transaction(2) {|snapshot|
141
+ while (do_read.running?)
142
+ p [ 2, snapshot ] if $DEBUG
143
+ assert_equal([ :foo ], snapshot.keys(store))
144
+ assert_equal(true, (snapshot.key? store, :foo))
145
+ assert_equal('I like ruby.', snapshot.fetch(:foo, :d) { store[:foo] })
146
+ update_read_count.succ!
147
+ update_read_latch.start if (update_read_count.value == MVCC_WARMUP_COUNT)
148
+ end
149
+ }
150
+ }
151
+ update_read_latch.wait
152
+
153
+ # delete
154
+ @cache.write_old_values(2, [ [ :value, :foo, :d, store[:foo] ] ])
155
+ store.delete(:foo)
156
+
157
+ # change number: 3
158
+ delete_read_latch = Latch.new
159
+ delete_read_count = Count.new
160
+ delete_read_thread = Thread.new{
161
+ @cache.transaction(3) {|snapshot|
162
+ while (do_read.running?)
163
+ p [ 3, snapshot ] if $DEBUG
164
+ assert_equal([], snapshot.keys(store))
165
+ assert_equal(false, (snapshot.key? store, :foo))
166
+ assert_nil(snapshot.fetch(:foo, :d) { store[:foo] })
167
+ delete_read_count.succ!
168
+ delete_read_latch.start if (delete_read_count.value == MVCC_WARMUP_COUNT)
169
+ end
170
+ }
171
+ }
172
+ delete_read_latch.wait
173
+
174
+ # insert
175
+ @cache.write_old_values(3, [ [ :none, :foo ] ])
176
+ store[:foo] = 'Do you like ruby?'
177
+
178
+ [ init_read_count,
179
+ insert_read_count,
180
+ update_read_count,
181
+ delete_read_count,
182
+ ].each do |count|
183
+ c = count.value
184
+ until (count.value > c)
185
+ # nothing to do.
186
+ end
187
+ end
188
+
189
+ do_read.running = false
190
+ [ init_read_thread,
191
+ insert_read_thread,
192
+ update_read_thread,
193
+ delete_read_thread
194
+ ].each do |thread|
195
+ thread.join
196
+ end
197
+ end
198
+ end
199
+
200
+ class MVCCCacheWithStorageTest < Test::Unit::TestCase
201
+ include Higgs
202
+
203
+ # for ident(1)
204
+ CVS_ID = '$Id: test_cache_mvcc.rb 742 2008-01-13 15:55:53Z toki $'
205
+
206
+ def setup # preset for rand
207
+ srand(0)
208
+ @cache = MVCCCache.new
209
+
210
+ @test_dir = 'st_test'
211
+ FileUtils.rm_rf(@test_dir) # for debug
212
+ FileUtils.mkdir_p(@test_dir)
213
+ @name = File.join(@test_dir, 'foo')
214
+ @logger = proc{|path|
215
+ logger = Logger.new(path, 1)
216
+ logger.level = Logger::DEBUG
217
+ logger
218
+ }
219
+ @st = Storage.new(@name, :logger => @logger)
220
+ end
221
+
222
+ def teardown
223
+ @st.shutdown unless @st.shutdown?
224
+ FileUtils.rm_rf(@test_dir) unless $DEBUG
225
+ end
226
+
227
+ class RunFlag
228
+ def initialize(running)
229
+ #@lock = Mutex.new
230
+ @running = running
231
+ end
232
+
233
+ def running=(running)
234
+ #@lock.synchronize{
235
+ @running = running
236
+ #}
237
+ end
238
+
239
+ def running?
240
+ #@lock.synchronize{
241
+ @running
242
+ #}
243
+ end
244
+ end
245
+
246
+ NUM_OF_READ_THREADS = 10
247
+ NUM_OF_DATA_CHUNK = 100
248
+ HEAT_RUN_TIME = 60
249
+ STORAGE_ITEMS = 1000
250
+ MAX_ITEM_BYTES = 1024*5
251
+ ITEM_CHARS = ('A'..'Z').to_a + ('a'..'z').to_a
252
+
253
+ def update_storage
254
+ NUM_OF_DATA_CHUNK.times do
255
+ write_list = []
256
+ ope = [ :write, :system_properties, :custom_properties, :delete ][rand(4)]
257
+ key = rand(STORAGE_ITEMS)
258
+ case (ope)
259
+ when :write
260
+ value = rand(256).chr * rand(MAX_ITEM_BYTES)
261
+ write_list << [ ope, key, value ]
262
+ when :system_properties
263
+ next unless (@st.key? key)
264
+ write_list << [ ope, key, { 'string_only' => [ true, false ][rand(2)] } ]
265
+ when :custom_properties
266
+ next unless (@st.key? key)
267
+ value = ITEM_CHARS[rand(ITEM_CHARS.length)] * rand(MAX_ITEM_BYTES)
268
+ write_list << [ ope, key, { 'foo' => value } ]
269
+ when :delete
270
+ next unless (@st.key? key)
271
+ write_list << [ ope, key ]
272
+ else
273
+ raise "unknown operation: #{ope}"
274
+ end
275
+
276
+ if (block_given?) then
277
+ old_values = []
278
+ for ope, key, value in write_list
279
+ case (ope)
280
+ when :write
281
+ if (@st.key? key) then
282
+ properties = Marshal.load(Marshal.dump(@st.fetch_properties(key)))
283
+ old_values << [ :value, key, :data, @st.fetch(key) ]
284
+ old_values << [ :value, key, :properties, properties ]
285
+ old_values << [ :value, key, :data_change_number, @st.data_change_number(key) ]
286
+ old_values << [ :value, key, :properties_change_number, @st.properties_change_number(key) ]
287
+ old_values << [ :value, key, :unique_data_id, @st.unique_data_id(key) ]
288
+ else
289
+ old_values << [ :none, key ]
290
+ end
291
+ when :system_properties, :custom_properties
292
+ if (@st.key? key) then
293
+ properties = Marshal.load(Marshal.dump(@st.fetch_properties(key)))
294
+ old_values << [ :value, key, :properties, properties ]
295
+ old_values << [ :value, key, :properties_change_number, @st.properties_change_number(key) ]
296
+ else
297
+ old_values << [ :none, key ]
298
+ end
299
+ when :delete
300
+ if (@st.key? key) then
301
+ properties = Marshal.load(Marshal.dump(@st.fetch_properties(key)))
302
+ old_values << [ :value, key, :data, @st.fetch(key) ]
303
+ old_values << [ :value, key, :properties, properties ]
304
+ old_values << [ :value, key, :data_change_number, @st.data_change_number(key) ]
305
+ old_values << [ :value, key, :properties_change_number, @st.properties_change_number(key) ]
306
+ old_values << [ :value, key, :unique_data_id, @st.unique_data_id(key) ]
307
+ end
308
+ end
309
+ end
310
+ yield(@st.change_number, old_values)
311
+ end
312
+
313
+ @st.write_and_commit(write_list)
314
+ end
315
+ end
316
+ private :update_storage
317
+
318
+ def test_mvcc_random_write
319
+ do_read = RunFlag.new(true)
320
+ update_storage
321
+
322
+ init_cnum = @st.change_number
323
+ init_keys = @st.keys.sort
324
+ init_values = {}
325
+ for key in init_keys
326
+ init_values[key] = {
327
+ :data => @st.fetch(key),
328
+ :properties => Marshal.load(Marshal.dump(@st.fetch_properties(key))),
329
+ :data_change_number => @st.data_change_number(key),
330
+ :properties_change_number => @st.properties_change_number(key),
331
+ :unique_data_id => @st.unique_data_id(key)
332
+ }
333
+ end
334
+
335
+ th_read_grp = ThreadGroup.new
336
+ read_start_latch = CountDownLatch.new(NUM_OF_READ_THREADS)
337
+ NUM_OF_READ_THREADS.times{|i| # `i' should be local scope of thread block
338
+ th_read_grp.add Thread.new{
339
+ @cache.transaction(init_cnum) {|snapshot|
340
+ read_start_latch.count_down
341
+
342
+ while (do_read.running?)
343
+ assert_equal(init_keys, snapshot.keys(@st).sort, "keys at thread:#{i}")
344
+ STORAGE_ITEMS.times do |k|
345
+ assert_equal((init_values.key? k),
346
+ (snapshot.key? @st, k), "thread: #{i}, key: #{k}")
347
+ assert_equal(init_values[k] && init_values[k][:data],
348
+ snapshot.fetch(k, :data) { @st.fetch(k) }, "thread: #{i}, key: #{k}")
349
+ assert_equal(init_values[k] && init_values[k][:properties],
350
+ snapshot.fetch(k, :properties) { @st.fetch_properties(k) }, "thread: #{i}, key: #{k}")
351
+ assert_equal(init_values[k] && init_values[k][:data_change_number],
352
+ snapshot.fetch(k, :data_change_number) { @st.data_change_number(k) }, "thread: #{i}, key: #{k}")
353
+ assert_equal(init_values[k] && init_values[k][:properties_change_number],
354
+ snapshot.fetch(k, :properties_change_number) { @st.properties_change_number(k) }, "thread: #{i}, key: #{k}")
355
+ assert_equal(init_values[k] && init_values[k][:unique_data_id],
356
+ snapshot.fetch(k, :unique_data_id) { @st.unique_data_id(k) }, "thread: #{i}, key: #{k}")
357
+ end
358
+ end
359
+ }
360
+ }
361
+ }
362
+ read_start_latch.wait
363
+
364
+ t0 = Time.now
365
+ while (Time.now - t0 < HEAT_RUN_TIME)
366
+ update_storage{|cnum, write_list|
367
+ @cache.write_old_values(cnum, write_list)
368
+ }
369
+ end
370
+
371
+ sleep(HEAT_RUN_TIME)
372
+ do_read.running = false
373
+ for t in th_read_grp.list
374
+ t.join
375
+ end
376
+ end
377
+
378
+ WORK_COUNT = 100
379
+ NUM_OF_READ_WRITE_THREADS = 10
380
+
381
+ def test_mvcc_frequency_update
382
+ do_read = RunFlag.new(true)
383
+ write_lock = Mutex.new
384
+
385
+ th_read_grp = ThreadGroup.new
386
+ read_start_latch = CountDownLatch.new(NUM_OF_READ_WRITE_THREADS)
387
+ NUM_OF_READ_WRITE_THREADS.times{|i| # `i' should be local scope of thread block
388
+ th_read_grp.add Thread.new{
389
+ read_start_latch.count_down
390
+
391
+ while (do_read.running?)
392
+ @cache.transaction(@st.change_number) {|snapshot|
393
+ if (snapshot.change_number > 0) then
394
+ assert_equal(snapshot.change_number.to_s,
395
+ snapshot.fetch(:foo, :data) { @st.fetch(:foo) }, "thread: #{i}")
396
+ else
397
+ assert_nil(snapshot.fetch(:foo, :data) { @st.fetch(:foo) }, "thread: #{i}")
398
+ end
399
+ }
400
+ end
401
+ }
402
+ }
403
+ read_start_latch.wait
404
+
405
+ th_write_grp = ThreadGroup.new
406
+ write_start_latch = CountDownLatch.new(NUM_OF_READ_WRITE_THREADS)
407
+ NUM_OF_READ_WRITE_THREADS.times{|i| # `i' should be local scope of thread block
408
+ th_write_grp.add Thread.new{
409
+ write_start_latch.count_down
410
+
411
+ WORK_COUNT.times do |j|
412
+ write_lock.synchronize{
413
+ @cache.transaction(@st.change_number) {|snapshot|
414
+ value = snapshot.fetch(:foo, :data) { @st.fetch(:foo) }
415
+ if (snapshot.change_number > 0) then
416
+ assert_equal(snapshot.change_number.to_s, value, "thread: #{i}-#{j}")
417
+ else
418
+ assert_nil(value, "thread: #{i}-#{j}")
419
+ end
420
+
421
+ next_value = (value || '0').succ
422
+
423
+ snapshot.write_old_values([ [ :value, :foo, :data, value ] ])
424
+ @st.write_and_commit([ [ :write, :foo, next_value ] ])
425
+ }
426
+ }
427
+ end
428
+ }
429
+ }
430
+ write_start_latch.wait
431
+
432
+ for t in th_write_grp.list
433
+ t.join
434
+ end
435
+
436
+ do_read.running = false
437
+ for t in th_read_grp.list
438
+ t.join
439
+ end
440
+
441
+ assert_equal((NUM_OF_READ_WRITE_THREADS * WORK_COUNT).to_s, @st.fetch(:foo))
442
+ end
443
+ end
444
+ end