higgs 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (64) hide show
  1. data/ChangeLog +208 -0
  2. data/LICENSE +26 -0
  3. data/README +2 -0
  4. data/Rakefile +75 -0
  5. data/bin/higgs_backup +67 -0
  6. data/bin/higgs_dump_index +43 -0
  7. data/bin/higgs_dump_jlog +42 -0
  8. data/bin/higgs_verify +37 -0
  9. data/lib/cgi/session/higgs.rb +72 -0
  10. data/lib/higgs/block.rb +192 -0
  11. data/lib/higgs/cache.rb +117 -0
  12. data/lib/higgs/dbm.rb +55 -0
  13. data/lib/higgs/exceptions.rb +31 -0
  14. data/lib/higgs/flock.rb +77 -0
  15. data/lib/higgs/index.rb +164 -0
  16. data/lib/higgs/jlog.rb +159 -0
  17. data/lib/higgs/lock.rb +189 -0
  18. data/lib/higgs/storage.rb +1086 -0
  19. data/lib/higgs/store.rb +228 -0
  20. data/lib/higgs/tar.rb +390 -0
  21. data/lib/higgs/thread.rb +370 -0
  22. data/lib/higgs/tman.rb +513 -0
  23. data/lib/higgs/utils/bman.rb +285 -0
  24. data/lib/higgs/utils.rb +22 -0
  25. data/lib/higgs/version.rb +21 -0
  26. data/lib/higgs.rb +59 -0
  27. data/misc/cache_bench/cache_bench.rb +43 -0
  28. data/misc/dbm_bench/.strc +8 -0
  29. data/misc/dbm_bench/Rakefile +78 -0
  30. data/misc/dbm_bench/dbm_multi_thread.rb +199 -0
  31. data/misc/dbm_bench/dbm_rnd_delete.rb +43 -0
  32. data/misc/dbm_bench/dbm_rnd_read.rb +44 -0
  33. data/misc/dbm_bench/dbm_rnd_update.rb +44 -0
  34. data/misc/dbm_bench/dbm_seq_read.rb +45 -0
  35. data/misc/dbm_bench/dbm_seq_write.rb +44 -0
  36. data/misc/dbm_bench/st_verify.rb +28 -0
  37. data/misc/io_bench/cksum_bench.rb +48 -0
  38. data/misc/io_bench/jlog_bench.rb +71 -0
  39. data/misc/io_bench/write_bench.rb +128 -0
  40. data/misc/thread_bench/lock_bench.rb +132 -0
  41. data/mkrdoc.rb +8 -0
  42. data/rdoc.yml +13 -0
  43. data/sample/count.rb +60 -0
  44. data/sample/dbmtest.rb +38 -0
  45. data/test/Rakefile +45 -0
  46. data/test/run.rb +32 -0
  47. data/test/test_block.rb +163 -0
  48. data/test/test_cache.rb +214 -0
  49. data/test/test_cgi_session.rb +142 -0
  50. data/test/test_flock.rb +162 -0
  51. data/test/test_index.rb +258 -0
  52. data/test/test_jlog.rb +180 -0
  53. data/test/test_lock.rb +320 -0
  54. data/test/test_online_backup.rb +169 -0
  55. data/test/test_storage.rb +439 -0
  56. data/test/test_storage_conf.rb +202 -0
  57. data/test/test_storage_init_opts.rb +89 -0
  58. data/test/test_store.rb +211 -0
  59. data/test/test_tar.rb +432 -0
  60. data/test/test_thread.rb +541 -0
  61. data/test/test_tman.rb +875 -0
  62. data/test/test_tman_init_opts.rb +56 -0
  63. data/test/test_utils_bman.rb +234 -0
  64. metadata +115 -0
data/lib/higgs/tman.rb ADDED
@@ -0,0 +1,513 @@
1
+ # = transaction manager
2
+ #
3
+ # Author:: $Author: toki $
4
+ # Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
5
+ # Revision:: $Revision: 559 $
6
+ #
7
+ # == license
8
+ # :include:LICENSE
9
+ #
10
+
11
+ require 'forwardable'
12
+ require 'higgs/cache'
13
+ require 'higgs/exceptions'
14
+ require 'higgs/lock'
15
+ require 'higgs/storage'
16
+ require 'singleton'
17
+
18
+ module Higgs
19
+ class TransactionManager
20
+ # for ident(1)
21
+ CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
22
+
23
+ include Exceptions
24
+
25
+ class Error < HiggsError
26
+ end
27
+
28
+ class NotWritableError < Error
29
+ end
30
+
31
+ class PseudoSecondaryCache
32
+ include Singleton
33
+
34
+ def [](key)
35
+ end
36
+
37
+ def []=(key, value)
38
+ value
39
+ end
40
+
41
+ def delete(key)
42
+ end
43
+ end
44
+
45
+ # options for Higgs::TransactionManager
46
+ module InitOptions
47
+ # these options are defined.
48
+ # [<tt>:read_only</tt>] if <tt>true</tt> then transaction is always read-only.
49
+ # default is <tt>false</tt>.
50
+ # [<tt>:decode</tt>] procedure to decode data on read. if <tt>:string_only</tt>
51
+ # property at an entry is <tt>true</tt> then <tt>decode</tt>
52
+ # is not used to read the entry. default is <tt>proc{|r| r }</tt>.
53
+ # [<tt>:encode</tt>] procedure to encode data on write. if <tt>:string_only</tt>
54
+ # property at an entry is <tt>true</tt> then <tt>encode</tt>
55
+ # is not used to write the entry. default is <tt>proc{|w| w }</tt>.
56
+ # [<tt>:lock_manager</tt>] lock of a transaction and individual data. default is
57
+ # a new instance of Higgs::GiantLockManager.
58
+ # [<tt>:master_cache</tt>] read-cache for encoded string data. defauilt is
59
+ # a new instance of Higgs::LRUCache.
60
+ # [<tt>:secondary_cache</tt>] secondary read-cache for encoded string data.
61
+ # key of cache is always unique string.
62
+ # default is no effect.
63
+ def init_options(options)
64
+ if (options.key? :read_only) then
65
+ @read_only = options[:read_only]
66
+ else
67
+ @read_only = false
68
+ end
69
+
70
+ @decode = options[:decode] || proc{|r| r }
71
+ @encode = options[:encode] || proc{|w| w }
72
+ @lock_manager = options[:lock_manager] || GiantLockManager.new
73
+ @master_cache = options[:master_cache] || LRUCache.new
74
+ @secondary_cache = options[:secondary_cache] || PseudoSecondaryCache.instance
75
+ end
76
+
77
+ attr_reader :read_only
78
+ end
79
+ include InitOptions
80
+
81
+ # export Higgs::TransactionManager methods from <tt>@tman</tt> instance variable.
82
+ #
83
+ # these methods are delegated.
84
+ # * Higgs::TransactionManager#read_only
85
+ # * Higgs::TransactionManager#transaction
86
+ #
87
+ module Export
88
+ extend Forwardable
89
+
90
+ def_delegator :@tman, :read_only
91
+ def_delegator :@tman, :transaction
92
+ end
93
+
94
+ def initialize(storage, options={})
95
+ @storage = storage
96
+ init_options(options)
97
+ @master_cache = SharedWorkCache.new(@master_cache) {|key|
98
+ (id = @storage.identity(key) and @secondary_cache[id]) or
99
+ (value = @storage.fetch(key) and @secondary_cache[@storage.identity(key)] = value.freeze)
100
+ }
101
+ end
102
+
103
+ # <tt>tx</tt> of block argument is transaction context and see
104
+ # Higgs::TransactionContext for detail.
105
+ def transaction(read_only=@read_only)
106
+ r = nil
107
+ @lock_manager.transaction(read_only) {|lock_handler|
108
+ begin
109
+ if (TransactionManager.in_transaction?) then
110
+ raise 'nested transaction forbidden'
111
+ end
112
+ if (read_only) then
113
+ tx = ReadOnlyTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
114
+ else
115
+ if (@read_only) then
116
+ raise NotWritableError, 'not writable'
117
+ end
118
+ tx = ReadWriteTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
119
+ end
120
+ Thread.current[:higgs_current_transaction] = tx
121
+ r = yield(tx)
122
+ tx.commit unless read_only
123
+ ensure
124
+ Thread.current[:higgs_current_transaction] = nil
125
+ end
126
+ }
127
+ r
128
+ end
129
+
130
+ def self.in_transaction?
131
+ ! Thread.current[:higgs_current_transaction].nil?
132
+ end
133
+
134
+ def self.current_transaction
135
+ Thread.current[:higgs_current_transaction]
136
+ end
137
+ end
138
+
139
+ class TransactionContext
140
+ # for ident(1)
141
+ CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
142
+
143
+ include Enumerable
144
+
145
+ def deep_copy(obj)
146
+ Marshal.load(Marshal.dump(obj))
147
+ end
148
+ private :deep_copy
149
+
150
+ def initialize(lock_handler, storage, master_cache, secondary_cache, decode, encode)
151
+ @lock_handler = lock_handler
152
+ @storage = storage
153
+ @master_cache = master_cache
154
+ @secondary_cache = secondary_cache
155
+ @decode = decode
156
+ @encode = encode
157
+
158
+ @local_data_cache = Hash.new{|hash, key|
159
+ if (@storage.key? key) then
160
+ if (@storage.string_only(key)) then
161
+ hash[key] = @master_cache[key]
162
+ else
163
+ hash[key] = @decode.call(@master_cache[key])
164
+ end
165
+ end
166
+ }
167
+
168
+ @local_properties_cache = Hash.new{|hash, key|
169
+ if (properties = @storage.fetch_properties(key)) then
170
+ hash[key] = deep_copy(properties)
171
+ else
172
+ hash[key] = { 'system_properties' => {}, 'custom_properties' => {} }
173
+ end
174
+ }
175
+
176
+ @locked_map = {}
177
+ @locked_map.default = false
178
+ @update_system_properties = {}
179
+ @update_custom_properties = {}
180
+ @ope_map = {}
181
+ end
182
+
183
+ def locked?(key)
184
+ @locked_map[key]
185
+ end
186
+
187
+ def lock(key)
188
+ unless (@locked_map[key]) then
189
+ @lock_handler.lock(key)
190
+ @locked_map[key] = true
191
+ end
192
+ nil
193
+ end
194
+
195
+ def unlock(key)
196
+ if (@locked_map[key]) then
197
+ @lock_handler.unlock(key)
198
+ @locked_map[key] = false
199
+ end
200
+ nil
201
+ end
202
+
203
+ def [](key)
204
+ lock(key)
205
+ (@ope_map[key] != :delete) ? @local_data_cache[key] : nil
206
+ end
207
+
208
+ def []=(key, value)
209
+ lock(key)
210
+ @ope_map[key] = :write
211
+ @local_data_cache[key] = value
212
+ end
213
+
214
+ def update(key, default_value=nil)
215
+ if (self.key? key) then # lock
216
+ value = self[key]
217
+ else
218
+ unless (default_value) then
219
+ raise IndexError, "not exist properties at key: #{key}"
220
+ end
221
+ value = default_value
222
+ end
223
+ r = yield(value)
224
+ self[key] = value
225
+ r
226
+ end
227
+
228
+ def delete(key)
229
+ lock(key)
230
+ if (@ope_map[key] != :delete) then
231
+ @ope_map[key] = :delete
232
+ @update_system_properties.delete(key)
233
+ @update_custom_properties.delete(key)
234
+ @local_properties_cache.delete(key)
235
+ @local_data_cache[key] # data load from storage
236
+ @local_data_cache.delete(key)
237
+ end
238
+ end
239
+
240
+ def key?(key)
241
+ lock(key)
242
+ if (@ope_map[key] != :delete) then
243
+ if (@local_data_cache.key? key) then
244
+ return true
245
+ end
246
+ if (@storage.key? key) then
247
+ return true
248
+ end
249
+ end
250
+ false
251
+ end
252
+
253
+ alias has_key? key?
254
+ alias include? key?
255
+ alias root? key?
256
+
257
+ def each_key
258
+ @local_data_cache.each_key do |key|
259
+ lock(key)
260
+ if (@ope_map[key] != :delete) then
261
+ yield(key)
262
+ end
263
+ end
264
+ @storage.each_key do |key|
265
+ lock(key)
266
+ if (@ope_map[key] != :delete) then
267
+ unless (@local_data_cache.key? key) then
268
+ yield(key)
269
+ end
270
+ end
271
+ end
272
+ self
273
+ end
274
+
275
+ def each_value # :yields: value
276
+ each_key do |key|
277
+ yield(self[key])
278
+ end
279
+ end
280
+
281
+ def each_pair # :yields: key, value
282
+ each_key do |key|
283
+ yield(key, self[key])
284
+ end
285
+ end
286
+
287
+ alias each each_pair
288
+
289
+ def keys
290
+ key_list = []
291
+ each_key do |key|
292
+ key_list << key
293
+ end
294
+ key_list
295
+ end
296
+
297
+ alias roots keys
298
+
299
+ def values
300
+ value_list = []
301
+ each_value do |value|
302
+ value_list << value
303
+ end
304
+ value_list
305
+ end
306
+
307
+ def length
308
+ len = 0
309
+ each_key do |key|
310
+ len += 1
311
+ end
312
+ len
313
+ end
314
+
315
+ alias size length
316
+
317
+ def empty?
318
+ length == 0
319
+ end
320
+
321
+ def property(key, name)
322
+ case (name)
323
+ when Symbol, String
324
+ # good
325
+ else
326
+ raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
327
+ end
328
+
329
+ lock(key)
330
+ if (@ope_map[key] != :delete) then
331
+ if (properties = @local_properties_cache[key]) then
332
+ case (name)
333
+ when Symbol
334
+ properties['system_properties'][name.to_s]
335
+ when String
336
+ properties['custom_properties'][name]
337
+ end
338
+ end
339
+ end
340
+ end
341
+
342
+ def set_property(key, name, value)
343
+ unless (self.key? key) then # lock
344
+ raise IndexError, "not exist properties at key: #{key}"
345
+ end
346
+ case (name)
347
+ when String
348
+ properties = @local_properties_cache[key]['custom_properties']
349
+ properties[name] = value
350
+ @update_custom_properties[key] = properties
351
+ when :string_only
352
+ properties = @local_properties_cache[key]['system_properties']
353
+ properties['string_only'] = value
354
+ @update_system_properties[key] = properties
355
+ else
356
+ raise TypeError, "can't convert #{name.class} (name) to String"
357
+ end
358
+ nil
359
+ end
360
+
361
+ def delete_property(key, name)
362
+ unless (self.key? key) then # lock
363
+ raise IndexError, "not exist properties at key: #{key}"
364
+ end
365
+ unless (name.kind_of? String) then
366
+ raise TypeError, "can't convert #{name.class} (name) to String"
367
+ end
368
+ properties = @local_properties_cache[key]['custom_properties']
369
+ if (properties.key? name) then
370
+ value = properties.delete(name)
371
+ @update_custom_properties[key] = properties
372
+ return value
373
+ end
374
+ nil
375
+ end
376
+
377
+ def property?(key, name)
378
+ case (name)
379
+ when Symbol, String
380
+ # good
381
+ else
382
+ raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
383
+ end
384
+
385
+ if (self.key? key) then # lock
386
+ case (name)
387
+ when Symbol
388
+ return (@local_properties_cache[key]['system_properties'].key? name.to_s)
389
+ when String
390
+ return (@local_properties_cache[key]['custom_properties'].key? name)
391
+ else
392
+ raise 'Bug: not to reach'
393
+ end
394
+ end
395
+ false
396
+ end
397
+
398
+ def each_property(key) # :yields: name, value
399
+ unless (self.key? key) then # lock
400
+ raise IndexError, "not exist properties at key: #{key}"
401
+ end
402
+ @local_properties_cache[key]['system_properties'].each_pair do |name, value|
403
+ yield(name.to_sym, value)
404
+ end
405
+ @local_properties_cache[key]['custom_properties'].each_pair do |name, value|
406
+ yield(name, value)
407
+ end
408
+ self
409
+ end
410
+
411
+ def delete_if(*keys) # :yields: key, value
412
+ del_list = []
413
+ if (keys.empty?) then
414
+ each_key do |key|
415
+ if (yield(key, self[key])) then
416
+ del_list << key
417
+ end
418
+ end
419
+ else
420
+ for key in keys
421
+ if (self.key? key) then
422
+ if (yield(key, self[key])) then
423
+ del_list << key
424
+ end
425
+ end
426
+ end
427
+ end
428
+ for key in del_list
429
+ delete(key)
430
+ end
431
+ nil
432
+ end
433
+
434
+ def clear
435
+ for key in keys
436
+ delete(key)
437
+ end
438
+ nil
439
+ end
440
+
441
+ def write_list
442
+ @ope_map.map{|key, ope|
443
+ if (ope == :delete) then
444
+ [ ope, key ]
445
+ else
446
+ if (@local_properties_cache[key]['system_properties']['string_only']) then
447
+ [ ope, key, @local_data_cache[key] ]
448
+ else
449
+ [ ope, key, @encode.call(@local_data_cache[key]) ]
450
+ end
451
+ end
452
+ } + \
453
+ @update_system_properties.map{|key, properties|
454
+ [ :system_properties, key, properties ]
455
+ } + \
456
+ @update_custom_properties.map{|key, properties|
457
+ [ :custom_properties, key, properties ]
458
+ }
459
+ end
460
+ private :write_list
461
+
462
+ def commit
463
+ write_list = write_list()
464
+ unless (write_list.empty?) then
465
+ @storage.write_and_commit(write_list)
466
+ for ope, key, value in write_list
467
+ case (ope)
468
+ when :write
469
+ @master_cache.delete(key)
470
+ @secondary_cache[key] = value
471
+ when :delete
472
+ @master_cache.delete(key)
473
+ @secondary_cache.delete(key)
474
+ end
475
+ @local_properties_cache.delete(key)
476
+ end
477
+ @update_system_properties.clear
478
+ @update_custom_properties.clear
479
+ @ope_map.clear
480
+ end
481
+ nil
482
+ end
483
+
484
+ def rollback
485
+ @local_data_cache.clear
486
+ @local_properties_cache.clear
487
+ @update_system_properties.clear
488
+ @update_custom_properties.clear
489
+ @ope_map.clear
490
+ nil
491
+ end
492
+ end
493
+
494
+ class ReadOnlyTransactionContext < TransactionContext
495
+ undef []=
496
+ undef update
497
+ undef delete
498
+ undef set_property
499
+ undef delete_property
500
+ undef delete_if
501
+ undef clear
502
+ undef commit
503
+ undef rollback
504
+ end
505
+
506
+ class ReadWriteTransactionContext < TransactionContext
507
+ end
508
+ end
509
+
510
+ # Local Variables:
511
+ # mode: Ruby
512
+ # indent-tabs-mode: nil
513
+ # End: