higgs 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. data/ChangeLog +208 -0
  2. data/LICENSE +26 -0
  3. data/README +2 -0
  4. data/Rakefile +75 -0
  5. data/bin/higgs_backup +67 -0
  6. data/bin/higgs_dump_index +43 -0
  7. data/bin/higgs_dump_jlog +42 -0
  8. data/bin/higgs_verify +37 -0
  9. data/lib/cgi/session/higgs.rb +72 -0
  10. data/lib/higgs/block.rb +192 -0
  11. data/lib/higgs/cache.rb +117 -0
  12. data/lib/higgs/dbm.rb +55 -0
  13. data/lib/higgs/exceptions.rb +31 -0
  14. data/lib/higgs/flock.rb +77 -0
  15. data/lib/higgs/index.rb +164 -0
  16. data/lib/higgs/jlog.rb +159 -0
  17. data/lib/higgs/lock.rb +189 -0
  18. data/lib/higgs/storage.rb +1086 -0
  19. data/lib/higgs/store.rb +228 -0
  20. data/lib/higgs/tar.rb +390 -0
  21. data/lib/higgs/thread.rb +370 -0
  22. data/lib/higgs/tman.rb +513 -0
  23. data/lib/higgs/utils/bman.rb +285 -0
  24. data/lib/higgs/utils.rb +22 -0
  25. data/lib/higgs/version.rb +21 -0
  26. data/lib/higgs.rb +59 -0
  27. data/misc/cache_bench/cache_bench.rb +43 -0
  28. data/misc/dbm_bench/.strc +8 -0
  29. data/misc/dbm_bench/Rakefile +78 -0
  30. data/misc/dbm_bench/dbm_multi_thread.rb +199 -0
  31. data/misc/dbm_bench/dbm_rnd_delete.rb +43 -0
  32. data/misc/dbm_bench/dbm_rnd_read.rb +44 -0
  33. data/misc/dbm_bench/dbm_rnd_update.rb +44 -0
  34. data/misc/dbm_bench/dbm_seq_read.rb +45 -0
  35. data/misc/dbm_bench/dbm_seq_write.rb +44 -0
  36. data/misc/dbm_bench/st_verify.rb +28 -0
  37. data/misc/io_bench/cksum_bench.rb +48 -0
  38. data/misc/io_bench/jlog_bench.rb +71 -0
  39. data/misc/io_bench/write_bench.rb +128 -0
  40. data/misc/thread_bench/lock_bench.rb +132 -0
  41. data/mkrdoc.rb +8 -0
  42. data/rdoc.yml +13 -0
  43. data/sample/count.rb +60 -0
  44. data/sample/dbmtest.rb +38 -0
  45. data/test/Rakefile +45 -0
  46. data/test/run.rb +32 -0
  47. data/test/test_block.rb +163 -0
  48. data/test/test_cache.rb +214 -0
  49. data/test/test_cgi_session.rb +142 -0
  50. data/test/test_flock.rb +162 -0
  51. data/test/test_index.rb +258 -0
  52. data/test/test_jlog.rb +180 -0
  53. data/test/test_lock.rb +320 -0
  54. data/test/test_online_backup.rb +169 -0
  55. data/test/test_storage.rb +439 -0
  56. data/test/test_storage_conf.rb +202 -0
  57. data/test/test_storage_init_opts.rb +89 -0
  58. data/test/test_store.rb +211 -0
  59. data/test/test_tar.rb +432 -0
  60. data/test/test_thread.rb +541 -0
  61. data/test/test_tman.rb +875 -0
  62. data/test/test_tman_init_opts.rb +56 -0
  63. data/test/test_utils_bman.rb +234 -0
  64. metadata +115 -0
data/lib/higgs/tman.rb ADDED
@@ -0,0 +1,513 @@
1
+ # = transaction manager
2
+ #
3
+ # Author:: $Author: toki $
4
+ # Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
5
+ # Revision:: $Revision: 559 $
6
+ #
7
+ # == license
8
+ # :include:LICENSE
9
+ #
10
+
11
+ require 'forwardable'
12
+ require 'higgs/cache'
13
+ require 'higgs/exceptions'
14
+ require 'higgs/lock'
15
+ require 'higgs/storage'
16
+ require 'singleton'
17
+
18
+ module Higgs
19
+ class TransactionManager
20
+ # for ident(1)
21
+ CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
22
+
23
+ include Exceptions
24
+
25
+ class Error < HiggsError
26
+ end
27
+
28
+ class NotWritableError < Error
29
+ end
30
+
31
+ class PseudoSecondaryCache
32
+ include Singleton
33
+
34
+ def [](key)
35
+ end
36
+
37
+ def []=(key, value)
38
+ value
39
+ end
40
+
41
+ def delete(key)
42
+ end
43
+ end
44
+
45
+ # options for Higgs::TransactionManager
46
+ module InitOptions
47
+ # these options are defined.
48
+ # [<tt>:read_only</tt>] if <tt>true</tt> then transaction is always read-only.
49
+ # default is <tt>false</tt>.
50
+ # [<tt>:decode</tt>] procedure to decode data on read. if <tt>:string_only</tt>
51
+ # property at an entry is <tt>true</tt> then <tt>decode</tt>
52
+ # is not used to read the entry. default is <tt>proc{|r| r }</tt>.
53
+ # [<tt>:encode</tt>] procedure to encode data on write. if <tt>:string_only</tt>
54
+ # property at an entry is <tt>true</tt> then <tt>encode</tt>
55
+ # is not used to write the entry. default is <tt>proc{|w| w }</tt>.
56
+ # [<tt>:lock_manager</tt>] lock of a transaction and individual data. default is
57
+ # a new instance of Higgs::GiantLockManager.
58
+ # [<tt>:master_cache</tt>] read-cache for encoded string data. defauilt is
59
+ # a new instance of Higgs::LRUCache.
60
+ # [<tt>:secondary_cache</tt>] secondary read-cache for encoded string data.
61
+ # key of cache is always unique string.
62
+ # default is no effect.
63
+ def init_options(options)
64
+ if (options.key? :read_only) then
65
+ @read_only = options[:read_only]
66
+ else
67
+ @read_only = false
68
+ end
69
+
70
+ @decode = options[:decode] || proc{|r| r }
71
+ @encode = options[:encode] || proc{|w| w }
72
+ @lock_manager = options[:lock_manager] || GiantLockManager.new
73
+ @master_cache = options[:master_cache] || LRUCache.new
74
+ @secondary_cache = options[:secondary_cache] || PseudoSecondaryCache.instance
75
+ end
76
+
77
+ attr_reader :read_only
78
+ end
79
+ include InitOptions
80
+
81
+ # export Higgs::TransactionManager methods from <tt>@tman</tt> instance variable.
82
+ #
83
+ # these methods are delegated.
84
+ # * Higgs::TransactionManager#read_only
85
+ # * Higgs::TransactionManager#transaction
86
+ #
87
+ module Export
88
+ extend Forwardable
89
+
90
+ def_delegator :@tman, :read_only
91
+ def_delegator :@tman, :transaction
92
+ end
93
+
94
+ def initialize(storage, options={})
95
+ @storage = storage
96
+ init_options(options)
97
+ @master_cache = SharedWorkCache.new(@master_cache) {|key|
98
+ (id = @storage.identity(key) and @secondary_cache[id]) or
99
+ (value = @storage.fetch(key) and @secondary_cache[@storage.identity(key)] = value.freeze)
100
+ }
101
+ end
102
+
103
+ # <tt>tx</tt> of block argument is transaction context and see
104
+ # Higgs::TransactionContext for detail.
105
+ def transaction(read_only=@read_only)
106
+ r = nil
107
+ @lock_manager.transaction(read_only) {|lock_handler|
108
+ begin
109
+ if (TransactionManager.in_transaction?) then
110
+ raise 'nested transaction forbidden'
111
+ end
112
+ if (read_only) then
113
+ tx = ReadOnlyTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
114
+ else
115
+ if (@read_only) then
116
+ raise NotWritableError, 'not writable'
117
+ end
118
+ tx = ReadWriteTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
119
+ end
120
+ Thread.current[:higgs_current_transaction] = tx
121
+ r = yield(tx)
122
+ tx.commit unless read_only
123
+ ensure
124
+ Thread.current[:higgs_current_transaction] = nil
125
+ end
126
+ }
127
+ r
128
+ end
129
+
130
+ def self.in_transaction?
131
+ ! Thread.current[:higgs_current_transaction].nil?
132
+ end
133
+
134
+ def self.current_transaction
135
+ Thread.current[:higgs_current_transaction]
136
+ end
137
+ end
138
+
139
+ class TransactionContext
140
+ # for ident(1)
141
+ CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
142
+
143
+ include Enumerable
144
+
145
+ def deep_copy(obj)
146
+ Marshal.load(Marshal.dump(obj))
147
+ end
148
+ private :deep_copy
149
+
150
+ def initialize(lock_handler, storage, master_cache, secondary_cache, decode, encode)
151
+ @lock_handler = lock_handler
152
+ @storage = storage
153
+ @master_cache = master_cache
154
+ @secondary_cache = secondary_cache
155
+ @decode = decode
156
+ @encode = encode
157
+
158
+ @local_data_cache = Hash.new{|hash, key|
159
+ if (@storage.key? key) then
160
+ if (@storage.string_only(key)) then
161
+ hash[key] = @master_cache[key]
162
+ else
163
+ hash[key] = @decode.call(@master_cache[key])
164
+ end
165
+ end
166
+ }
167
+
168
+ @local_properties_cache = Hash.new{|hash, key|
169
+ if (properties = @storage.fetch_properties(key)) then
170
+ hash[key] = deep_copy(properties)
171
+ else
172
+ hash[key] = { 'system_properties' => {}, 'custom_properties' => {} }
173
+ end
174
+ }
175
+
176
+ @locked_map = {}
177
+ @locked_map.default = false
178
+ @update_system_properties = {}
179
+ @update_custom_properties = {}
180
+ @ope_map = {}
181
+ end
182
+
183
+ def locked?(key)
184
+ @locked_map[key]
185
+ end
186
+
187
+ def lock(key)
188
+ unless (@locked_map[key]) then
189
+ @lock_handler.lock(key)
190
+ @locked_map[key] = true
191
+ end
192
+ nil
193
+ end
194
+
195
+ def unlock(key)
196
+ if (@locked_map[key]) then
197
+ @lock_handler.unlock(key)
198
+ @locked_map[key] = false
199
+ end
200
+ nil
201
+ end
202
+
203
+ def [](key)
204
+ lock(key)
205
+ (@ope_map[key] != :delete) ? @local_data_cache[key] : nil
206
+ end
207
+
208
+ def []=(key, value)
209
+ lock(key)
210
+ @ope_map[key] = :write
211
+ @local_data_cache[key] = value
212
+ end
213
+
214
+ def update(key, default_value=nil)
215
+ if (self.key? key) then # lock
216
+ value = self[key]
217
+ else
218
+ unless (default_value) then
219
+ raise IndexError, "not exist properties at key: #{key}"
220
+ end
221
+ value = default_value
222
+ end
223
+ r = yield(value)
224
+ self[key] = value
225
+ r
226
+ end
227
+
228
+ def delete(key)
229
+ lock(key)
230
+ if (@ope_map[key] != :delete) then
231
+ @ope_map[key] = :delete
232
+ @update_system_properties.delete(key)
233
+ @update_custom_properties.delete(key)
234
+ @local_properties_cache.delete(key)
235
+ @local_data_cache[key] # data load from storage
236
+ @local_data_cache.delete(key)
237
+ end
238
+ end
239
+
240
+ def key?(key)
241
+ lock(key)
242
+ if (@ope_map[key] != :delete) then
243
+ if (@local_data_cache.key? key) then
244
+ return true
245
+ end
246
+ if (@storage.key? key) then
247
+ return true
248
+ end
249
+ end
250
+ false
251
+ end
252
+
253
+ alias has_key? key?
254
+ alias include? key?
255
+ alias root? key?
256
+
257
+ def each_key
258
+ @local_data_cache.each_key do |key|
259
+ lock(key)
260
+ if (@ope_map[key] != :delete) then
261
+ yield(key)
262
+ end
263
+ end
264
+ @storage.each_key do |key|
265
+ lock(key)
266
+ if (@ope_map[key] != :delete) then
267
+ unless (@local_data_cache.key? key) then
268
+ yield(key)
269
+ end
270
+ end
271
+ end
272
+ self
273
+ end
274
+
275
+ def each_value # :yields: value
276
+ each_key do |key|
277
+ yield(self[key])
278
+ end
279
+ end
280
+
281
+ def each_pair # :yields: key, value
282
+ each_key do |key|
283
+ yield(key, self[key])
284
+ end
285
+ end
286
+
287
+ alias each each_pair
288
+
289
+ def keys
290
+ key_list = []
291
+ each_key do |key|
292
+ key_list << key
293
+ end
294
+ key_list
295
+ end
296
+
297
+ alias roots keys
298
+
299
+ def values
300
+ value_list = []
301
+ each_value do |value|
302
+ value_list << value
303
+ end
304
+ value_list
305
+ end
306
+
307
+ def length
308
+ len = 0
309
+ each_key do |key|
310
+ len += 1
311
+ end
312
+ len
313
+ end
314
+
315
+ alias size length
316
+
317
+ def empty?
318
+ length == 0
319
+ end
320
+
321
+ def property(key, name)
322
+ case (name)
323
+ when Symbol, String
324
+ # good
325
+ else
326
+ raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
327
+ end
328
+
329
+ lock(key)
330
+ if (@ope_map[key] != :delete) then
331
+ if (properties = @local_properties_cache[key]) then
332
+ case (name)
333
+ when Symbol
334
+ properties['system_properties'][name.to_s]
335
+ when String
336
+ properties['custom_properties'][name]
337
+ end
338
+ end
339
+ end
340
+ end
341
+
342
+ def set_property(key, name, value)
343
+ unless (self.key? key) then # lock
344
+ raise IndexError, "not exist properties at key: #{key}"
345
+ end
346
+ case (name)
347
+ when String
348
+ properties = @local_properties_cache[key]['custom_properties']
349
+ properties[name] = value
350
+ @update_custom_properties[key] = properties
351
+ when :string_only
352
+ properties = @local_properties_cache[key]['system_properties']
353
+ properties['string_only'] = value
354
+ @update_system_properties[key] = properties
355
+ else
356
+ raise TypeError, "can't convert #{name.class} (name) to String"
357
+ end
358
+ nil
359
+ end
360
+
361
+ def delete_property(key, name)
362
+ unless (self.key? key) then # lock
363
+ raise IndexError, "not exist properties at key: #{key}"
364
+ end
365
+ unless (name.kind_of? String) then
366
+ raise TypeError, "can't convert #{name.class} (name) to String"
367
+ end
368
+ properties = @local_properties_cache[key]['custom_properties']
369
+ if (properties.key? name) then
370
+ value = properties.delete(name)
371
+ @update_custom_properties[key] = properties
372
+ return value
373
+ end
374
+ nil
375
+ end
376
+
377
+ def property?(key, name)
378
+ case (name)
379
+ when Symbol, String
380
+ # good
381
+ else
382
+ raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
383
+ end
384
+
385
+ if (self.key? key) then # lock
386
+ case (name)
387
+ when Symbol
388
+ return (@local_properties_cache[key]['system_properties'].key? name.to_s)
389
+ when String
390
+ return (@local_properties_cache[key]['custom_properties'].key? name)
391
+ else
392
+ raise 'Bug: not to reach'
393
+ end
394
+ end
395
+ false
396
+ end
397
+
398
+ def each_property(key) # :yields: name, value
399
+ unless (self.key? key) then # lock
400
+ raise IndexError, "not exist properties at key: #{key}"
401
+ end
402
+ @local_properties_cache[key]['system_properties'].each_pair do |name, value|
403
+ yield(name.to_sym, value)
404
+ end
405
+ @local_properties_cache[key]['custom_properties'].each_pair do |name, value|
406
+ yield(name, value)
407
+ end
408
+ self
409
+ end
410
+
411
+ def delete_if(*keys) # :yields: key, value
412
+ del_list = []
413
+ if (keys.empty?) then
414
+ each_key do |key|
415
+ if (yield(key, self[key])) then
416
+ del_list << key
417
+ end
418
+ end
419
+ else
420
+ for key in keys
421
+ if (self.key? key) then
422
+ if (yield(key, self[key])) then
423
+ del_list << key
424
+ end
425
+ end
426
+ end
427
+ end
428
+ for key in del_list
429
+ delete(key)
430
+ end
431
+ nil
432
+ end
433
+
434
+ def clear
435
+ for key in keys
436
+ delete(key)
437
+ end
438
+ nil
439
+ end
440
+
441
+ def write_list
442
+ @ope_map.map{|key, ope|
443
+ if (ope == :delete) then
444
+ [ ope, key ]
445
+ else
446
+ if (@local_properties_cache[key]['system_properties']['string_only']) then
447
+ [ ope, key, @local_data_cache[key] ]
448
+ else
449
+ [ ope, key, @encode.call(@local_data_cache[key]) ]
450
+ end
451
+ end
452
+ } + \
453
+ @update_system_properties.map{|key, properties|
454
+ [ :system_properties, key, properties ]
455
+ } + \
456
+ @update_custom_properties.map{|key, properties|
457
+ [ :custom_properties, key, properties ]
458
+ }
459
+ end
460
+ private :write_list
461
+
462
+ def commit
463
+ write_list = write_list()
464
+ unless (write_list.empty?) then
465
+ @storage.write_and_commit(write_list)
466
+ for ope, key, value in write_list
467
+ case (ope)
468
+ when :write
469
+ @master_cache.delete(key)
470
+ @secondary_cache[key] = value
471
+ when :delete
472
+ @master_cache.delete(key)
473
+ @secondary_cache.delete(key)
474
+ end
475
+ @local_properties_cache.delete(key)
476
+ end
477
+ @update_system_properties.clear
478
+ @update_custom_properties.clear
479
+ @ope_map.clear
480
+ end
481
+ nil
482
+ end
483
+
484
+ def rollback
485
+ @local_data_cache.clear
486
+ @local_properties_cache.clear
487
+ @update_system_properties.clear
488
+ @update_custom_properties.clear
489
+ @ope_map.clear
490
+ nil
491
+ end
492
+ end
493
+
494
+ class ReadOnlyTransactionContext < TransactionContext
495
+ undef []=
496
+ undef update
497
+ undef delete
498
+ undef set_property
499
+ undef delete_property
500
+ undef delete_if
501
+ undef clear
502
+ undef commit
503
+ undef rollback
504
+ end
505
+
506
+ class ReadWriteTransactionContext < TransactionContext
507
+ end
508
+ end
509
+
510
+ # Local Variables:
511
+ # mode: Ruby
512
+ # indent-tabs-mode: nil
513
+ # End: