higgs 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +208 -0
- data/LICENSE +26 -0
- data/README +2 -0
- data/Rakefile +75 -0
- data/bin/higgs_backup +67 -0
- data/bin/higgs_dump_index +43 -0
- data/bin/higgs_dump_jlog +42 -0
- data/bin/higgs_verify +37 -0
- data/lib/cgi/session/higgs.rb +72 -0
- data/lib/higgs/block.rb +192 -0
- data/lib/higgs/cache.rb +117 -0
- data/lib/higgs/dbm.rb +55 -0
- data/lib/higgs/exceptions.rb +31 -0
- data/lib/higgs/flock.rb +77 -0
- data/lib/higgs/index.rb +164 -0
- data/lib/higgs/jlog.rb +159 -0
- data/lib/higgs/lock.rb +189 -0
- data/lib/higgs/storage.rb +1086 -0
- data/lib/higgs/store.rb +228 -0
- data/lib/higgs/tar.rb +390 -0
- data/lib/higgs/thread.rb +370 -0
- data/lib/higgs/tman.rb +513 -0
- data/lib/higgs/utils/bman.rb +285 -0
- data/lib/higgs/utils.rb +22 -0
- data/lib/higgs/version.rb +21 -0
- data/lib/higgs.rb +59 -0
- data/misc/cache_bench/cache_bench.rb +43 -0
- data/misc/dbm_bench/.strc +8 -0
- data/misc/dbm_bench/Rakefile +78 -0
- data/misc/dbm_bench/dbm_multi_thread.rb +199 -0
- data/misc/dbm_bench/dbm_rnd_delete.rb +43 -0
- data/misc/dbm_bench/dbm_rnd_read.rb +44 -0
- data/misc/dbm_bench/dbm_rnd_update.rb +44 -0
- data/misc/dbm_bench/dbm_seq_read.rb +45 -0
- data/misc/dbm_bench/dbm_seq_write.rb +44 -0
- data/misc/dbm_bench/st_verify.rb +28 -0
- data/misc/io_bench/cksum_bench.rb +48 -0
- data/misc/io_bench/jlog_bench.rb +71 -0
- data/misc/io_bench/write_bench.rb +128 -0
- data/misc/thread_bench/lock_bench.rb +132 -0
- data/mkrdoc.rb +8 -0
- data/rdoc.yml +13 -0
- data/sample/count.rb +60 -0
- data/sample/dbmtest.rb +38 -0
- data/test/Rakefile +45 -0
- data/test/run.rb +32 -0
- data/test/test_block.rb +163 -0
- data/test/test_cache.rb +214 -0
- data/test/test_cgi_session.rb +142 -0
- data/test/test_flock.rb +162 -0
- data/test/test_index.rb +258 -0
- data/test/test_jlog.rb +180 -0
- data/test/test_lock.rb +320 -0
- data/test/test_online_backup.rb +169 -0
- data/test/test_storage.rb +439 -0
- data/test/test_storage_conf.rb +202 -0
- data/test/test_storage_init_opts.rb +89 -0
- data/test/test_store.rb +211 -0
- data/test/test_tar.rb +432 -0
- data/test/test_thread.rb +541 -0
- data/test/test_tman.rb +875 -0
- data/test/test_tman_init_opts.rb +56 -0
- data/test/test_utils_bman.rb +234 -0
- metadata +115 -0
data/lib/higgs/tman.rb
ADDED
@@ -0,0 +1,513 @@
|
|
1
|
+
# = transaction manager
|
2
|
+
#
|
3
|
+
# Author:: $Author: toki $
|
4
|
+
# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
|
5
|
+
# Revision:: $Revision: 559 $
|
6
|
+
#
|
7
|
+
# == license
|
8
|
+
# :include:LICENSE
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'forwardable'
|
12
|
+
require 'higgs/cache'
|
13
|
+
require 'higgs/exceptions'
|
14
|
+
require 'higgs/lock'
|
15
|
+
require 'higgs/storage'
|
16
|
+
require 'singleton'
|
17
|
+
|
18
|
+
module Higgs
|
19
|
+
class TransactionManager
|
20
|
+
# for ident(1)
|
21
|
+
CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
|
22
|
+
|
23
|
+
include Exceptions
|
24
|
+
|
25
|
+
class Error < HiggsError
|
26
|
+
end
|
27
|
+
|
28
|
+
class NotWritableError < Error
|
29
|
+
end
|
30
|
+
|
31
|
+
class PseudoSecondaryCache
|
32
|
+
include Singleton
|
33
|
+
|
34
|
+
def [](key)
|
35
|
+
end
|
36
|
+
|
37
|
+
def []=(key, value)
|
38
|
+
value
|
39
|
+
end
|
40
|
+
|
41
|
+
def delete(key)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# options for Higgs::TransactionManager
|
46
|
+
module InitOptions
|
47
|
+
# these options are defined.
|
48
|
+
# [<tt>:read_only</tt>] if <tt>true</tt> then transaction is always read-only.
|
49
|
+
# default is <tt>false</tt>.
|
50
|
+
# [<tt>:decode</tt>] procedure to decode data on read. if <tt>:string_only</tt>
|
51
|
+
# property at an entry is <tt>true</tt> then <tt>decode</tt>
|
52
|
+
# is not used to read the entry. default is <tt>proc{|r| r }</tt>.
|
53
|
+
# [<tt>:encode</tt>] procedure to encode data on write. if <tt>:string_only</tt>
|
54
|
+
# property at an entry is <tt>true</tt> then <tt>encode</tt>
|
55
|
+
# is not used to write the entry. default is <tt>proc{|w| w }</tt>.
|
56
|
+
# [<tt>:lock_manager</tt>] lock of a transaction and individual data. default is
|
57
|
+
# a new instance of Higgs::GiantLockManager.
|
58
|
+
# [<tt>:master_cache</tt>] read-cache for encoded string data. defauilt is
|
59
|
+
# a new instance of Higgs::LRUCache.
|
60
|
+
# [<tt>:secondary_cache</tt>] secondary read-cache for encoded string data.
|
61
|
+
# key of cache is always unique string.
|
62
|
+
# default is no effect.
|
63
|
+
def init_options(options)
|
64
|
+
if (options.key? :read_only) then
|
65
|
+
@read_only = options[:read_only]
|
66
|
+
else
|
67
|
+
@read_only = false
|
68
|
+
end
|
69
|
+
|
70
|
+
@decode = options[:decode] || proc{|r| r }
|
71
|
+
@encode = options[:encode] || proc{|w| w }
|
72
|
+
@lock_manager = options[:lock_manager] || GiantLockManager.new
|
73
|
+
@master_cache = options[:master_cache] || LRUCache.new
|
74
|
+
@secondary_cache = options[:secondary_cache] || PseudoSecondaryCache.instance
|
75
|
+
end
|
76
|
+
|
77
|
+
attr_reader :read_only
|
78
|
+
end
|
79
|
+
include InitOptions
|
80
|
+
|
81
|
+
# export Higgs::TransactionManager methods from <tt>@tman</tt> instance variable.
|
82
|
+
#
|
83
|
+
# these methods are delegated.
|
84
|
+
# * Higgs::TransactionManager#read_only
|
85
|
+
# * Higgs::TransactionManager#transaction
|
86
|
+
#
|
87
|
+
module Export
|
88
|
+
extend Forwardable
|
89
|
+
|
90
|
+
def_delegator :@tman, :read_only
|
91
|
+
def_delegator :@tman, :transaction
|
92
|
+
end
|
93
|
+
|
94
|
+
def initialize(storage, options={})
|
95
|
+
@storage = storage
|
96
|
+
init_options(options)
|
97
|
+
@master_cache = SharedWorkCache.new(@master_cache) {|key|
|
98
|
+
(id = @storage.identity(key) and @secondary_cache[id]) or
|
99
|
+
(value = @storage.fetch(key) and @secondary_cache[@storage.identity(key)] = value.freeze)
|
100
|
+
}
|
101
|
+
end
|
102
|
+
|
103
|
+
# <tt>tx</tt> of block argument is transaction context and see
|
104
|
+
# Higgs::TransactionContext for detail.
|
105
|
+
def transaction(read_only=@read_only)
|
106
|
+
r = nil
|
107
|
+
@lock_manager.transaction(read_only) {|lock_handler|
|
108
|
+
begin
|
109
|
+
if (TransactionManager.in_transaction?) then
|
110
|
+
raise 'nested transaction forbidden'
|
111
|
+
end
|
112
|
+
if (read_only) then
|
113
|
+
tx = ReadOnlyTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
|
114
|
+
else
|
115
|
+
if (@read_only) then
|
116
|
+
raise NotWritableError, 'not writable'
|
117
|
+
end
|
118
|
+
tx = ReadWriteTransactionContext.new(lock_handler, @storage, @master_cache, @secondary_cache, @decode, @encode)
|
119
|
+
end
|
120
|
+
Thread.current[:higgs_current_transaction] = tx
|
121
|
+
r = yield(tx)
|
122
|
+
tx.commit unless read_only
|
123
|
+
ensure
|
124
|
+
Thread.current[:higgs_current_transaction] = nil
|
125
|
+
end
|
126
|
+
}
|
127
|
+
r
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.in_transaction?
|
131
|
+
! Thread.current[:higgs_current_transaction].nil?
|
132
|
+
end
|
133
|
+
|
134
|
+
def self.current_transaction
|
135
|
+
Thread.current[:higgs_current_transaction]
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
class TransactionContext
|
140
|
+
# for ident(1)
|
141
|
+
CVS_ID = '$Id: tman.rb 559 2007-09-25 15:20:20Z toki $'
|
142
|
+
|
143
|
+
include Enumerable
|
144
|
+
|
145
|
+
def deep_copy(obj)
|
146
|
+
Marshal.load(Marshal.dump(obj))
|
147
|
+
end
|
148
|
+
private :deep_copy
|
149
|
+
|
150
|
+
def initialize(lock_handler, storage, master_cache, secondary_cache, decode, encode)
|
151
|
+
@lock_handler = lock_handler
|
152
|
+
@storage = storage
|
153
|
+
@master_cache = master_cache
|
154
|
+
@secondary_cache = secondary_cache
|
155
|
+
@decode = decode
|
156
|
+
@encode = encode
|
157
|
+
|
158
|
+
@local_data_cache = Hash.new{|hash, key|
|
159
|
+
if (@storage.key? key) then
|
160
|
+
if (@storage.string_only(key)) then
|
161
|
+
hash[key] = @master_cache[key]
|
162
|
+
else
|
163
|
+
hash[key] = @decode.call(@master_cache[key])
|
164
|
+
end
|
165
|
+
end
|
166
|
+
}
|
167
|
+
|
168
|
+
@local_properties_cache = Hash.new{|hash, key|
|
169
|
+
if (properties = @storage.fetch_properties(key)) then
|
170
|
+
hash[key] = deep_copy(properties)
|
171
|
+
else
|
172
|
+
hash[key] = { 'system_properties' => {}, 'custom_properties' => {} }
|
173
|
+
end
|
174
|
+
}
|
175
|
+
|
176
|
+
@locked_map = {}
|
177
|
+
@locked_map.default = false
|
178
|
+
@update_system_properties = {}
|
179
|
+
@update_custom_properties = {}
|
180
|
+
@ope_map = {}
|
181
|
+
end
|
182
|
+
|
183
|
+
def locked?(key)
|
184
|
+
@locked_map[key]
|
185
|
+
end
|
186
|
+
|
187
|
+
def lock(key)
|
188
|
+
unless (@locked_map[key]) then
|
189
|
+
@lock_handler.lock(key)
|
190
|
+
@locked_map[key] = true
|
191
|
+
end
|
192
|
+
nil
|
193
|
+
end
|
194
|
+
|
195
|
+
def unlock(key)
|
196
|
+
if (@locked_map[key]) then
|
197
|
+
@lock_handler.unlock(key)
|
198
|
+
@locked_map[key] = false
|
199
|
+
end
|
200
|
+
nil
|
201
|
+
end
|
202
|
+
|
203
|
+
def [](key)
|
204
|
+
lock(key)
|
205
|
+
(@ope_map[key] != :delete) ? @local_data_cache[key] : nil
|
206
|
+
end
|
207
|
+
|
208
|
+
def []=(key, value)
|
209
|
+
lock(key)
|
210
|
+
@ope_map[key] = :write
|
211
|
+
@local_data_cache[key] = value
|
212
|
+
end
|
213
|
+
|
214
|
+
def update(key, default_value=nil)
|
215
|
+
if (self.key? key) then # lock
|
216
|
+
value = self[key]
|
217
|
+
else
|
218
|
+
unless (default_value) then
|
219
|
+
raise IndexError, "not exist properties at key: #{key}"
|
220
|
+
end
|
221
|
+
value = default_value
|
222
|
+
end
|
223
|
+
r = yield(value)
|
224
|
+
self[key] = value
|
225
|
+
r
|
226
|
+
end
|
227
|
+
|
228
|
+
def delete(key)
|
229
|
+
lock(key)
|
230
|
+
if (@ope_map[key] != :delete) then
|
231
|
+
@ope_map[key] = :delete
|
232
|
+
@update_system_properties.delete(key)
|
233
|
+
@update_custom_properties.delete(key)
|
234
|
+
@local_properties_cache.delete(key)
|
235
|
+
@local_data_cache[key] # data load from storage
|
236
|
+
@local_data_cache.delete(key)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def key?(key)
|
241
|
+
lock(key)
|
242
|
+
if (@ope_map[key] != :delete) then
|
243
|
+
if (@local_data_cache.key? key) then
|
244
|
+
return true
|
245
|
+
end
|
246
|
+
if (@storage.key? key) then
|
247
|
+
return true
|
248
|
+
end
|
249
|
+
end
|
250
|
+
false
|
251
|
+
end
|
252
|
+
|
253
|
+
alias has_key? key?
|
254
|
+
alias include? key?
|
255
|
+
alias root? key?
|
256
|
+
|
257
|
+
def each_key
|
258
|
+
@local_data_cache.each_key do |key|
|
259
|
+
lock(key)
|
260
|
+
if (@ope_map[key] != :delete) then
|
261
|
+
yield(key)
|
262
|
+
end
|
263
|
+
end
|
264
|
+
@storage.each_key do |key|
|
265
|
+
lock(key)
|
266
|
+
if (@ope_map[key] != :delete) then
|
267
|
+
unless (@local_data_cache.key? key) then
|
268
|
+
yield(key)
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|
272
|
+
self
|
273
|
+
end
|
274
|
+
|
275
|
+
def each_value # :yields: value
|
276
|
+
each_key do |key|
|
277
|
+
yield(self[key])
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
281
|
+
def each_pair # :yields: key, value
|
282
|
+
each_key do |key|
|
283
|
+
yield(key, self[key])
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
alias each each_pair
|
288
|
+
|
289
|
+
def keys
|
290
|
+
key_list = []
|
291
|
+
each_key do |key|
|
292
|
+
key_list << key
|
293
|
+
end
|
294
|
+
key_list
|
295
|
+
end
|
296
|
+
|
297
|
+
alias roots keys
|
298
|
+
|
299
|
+
def values
|
300
|
+
value_list = []
|
301
|
+
each_value do |value|
|
302
|
+
value_list << value
|
303
|
+
end
|
304
|
+
value_list
|
305
|
+
end
|
306
|
+
|
307
|
+
def length
|
308
|
+
len = 0
|
309
|
+
each_key do |key|
|
310
|
+
len += 1
|
311
|
+
end
|
312
|
+
len
|
313
|
+
end
|
314
|
+
|
315
|
+
alias size length
|
316
|
+
|
317
|
+
def empty?
|
318
|
+
length == 0
|
319
|
+
end
|
320
|
+
|
321
|
+
def property(key, name)
|
322
|
+
case (name)
|
323
|
+
when Symbol, String
|
324
|
+
# good
|
325
|
+
else
|
326
|
+
raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
|
327
|
+
end
|
328
|
+
|
329
|
+
lock(key)
|
330
|
+
if (@ope_map[key] != :delete) then
|
331
|
+
if (properties = @local_properties_cache[key]) then
|
332
|
+
case (name)
|
333
|
+
when Symbol
|
334
|
+
properties['system_properties'][name.to_s]
|
335
|
+
when String
|
336
|
+
properties['custom_properties'][name]
|
337
|
+
end
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|
341
|
+
|
342
|
+
def set_property(key, name, value)
|
343
|
+
unless (self.key? key) then # lock
|
344
|
+
raise IndexError, "not exist properties at key: #{key}"
|
345
|
+
end
|
346
|
+
case (name)
|
347
|
+
when String
|
348
|
+
properties = @local_properties_cache[key]['custom_properties']
|
349
|
+
properties[name] = value
|
350
|
+
@update_custom_properties[key] = properties
|
351
|
+
when :string_only
|
352
|
+
properties = @local_properties_cache[key]['system_properties']
|
353
|
+
properties['string_only'] = value
|
354
|
+
@update_system_properties[key] = properties
|
355
|
+
else
|
356
|
+
raise TypeError, "can't convert #{name.class} (name) to String"
|
357
|
+
end
|
358
|
+
nil
|
359
|
+
end
|
360
|
+
|
361
|
+
def delete_property(key, name)
|
362
|
+
unless (self.key? key) then # lock
|
363
|
+
raise IndexError, "not exist properties at key: #{key}"
|
364
|
+
end
|
365
|
+
unless (name.kind_of? String) then
|
366
|
+
raise TypeError, "can't convert #{name.class} (name) to String"
|
367
|
+
end
|
368
|
+
properties = @local_properties_cache[key]['custom_properties']
|
369
|
+
if (properties.key? name) then
|
370
|
+
value = properties.delete(name)
|
371
|
+
@update_custom_properties[key] = properties
|
372
|
+
return value
|
373
|
+
end
|
374
|
+
nil
|
375
|
+
end
|
376
|
+
|
377
|
+
def property?(key, name)
|
378
|
+
case (name)
|
379
|
+
when Symbol, String
|
380
|
+
# good
|
381
|
+
else
|
382
|
+
raise TypeError, "can't convert #{name.class} (name) to Symbol or String"
|
383
|
+
end
|
384
|
+
|
385
|
+
if (self.key? key) then # lock
|
386
|
+
case (name)
|
387
|
+
when Symbol
|
388
|
+
return (@local_properties_cache[key]['system_properties'].key? name.to_s)
|
389
|
+
when String
|
390
|
+
return (@local_properties_cache[key]['custom_properties'].key? name)
|
391
|
+
else
|
392
|
+
raise 'Bug: not to reach'
|
393
|
+
end
|
394
|
+
end
|
395
|
+
false
|
396
|
+
end
|
397
|
+
|
398
|
+
def each_property(key) # :yields: name, value
|
399
|
+
unless (self.key? key) then # lock
|
400
|
+
raise IndexError, "not exist properties at key: #{key}"
|
401
|
+
end
|
402
|
+
@local_properties_cache[key]['system_properties'].each_pair do |name, value|
|
403
|
+
yield(name.to_sym, value)
|
404
|
+
end
|
405
|
+
@local_properties_cache[key]['custom_properties'].each_pair do |name, value|
|
406
|
+
yield(name, value)
|
407
|
+
end
|
408
|
+
self
|
409
|
+
end
|
410
|
+
|
411
|
+
def delete_if(*keys) # :yields: key, value
|
412
|
+
del_list = []
|
413
|
+
if (keys.empty?) then
|
414
|
+
each_key do |key|
|
415
|
+
if (yield(key, self[key])) then
|
416
|
+
del_list << key
|
417
|
+
end
|
418
|
+
end
|
419
|
+
else
|
420
|
+
for key in keys
|
421
|
+
if (self.key? key) then
|
422
|
+
if (yield(key, self[key])) then
|
423
|
+
del_list << key
|
424
|
+
end
|
425
|
+
end
|
426
|
+
end
|
427
|
+
end
|
428
|
+
for key in del_list
|
429
|
+
delete(key)
|
430
|
+
end
|
431
|
+
nil
|
432
|
+
end
|
433
|
+
|
434
|
+
def clear
|
435
|
+
for key in keys
|
436
|
+
delete(key)
|
437
|
+
end
|
438
|
+
nil
|
439
|
+
end
|
440
|
+
|
441
|
+
def write_list
|
442
|
+
@ope_map.map{|key, ope|
|
443
|
+
if (ope == :delete) then
|
444
|
+
[ ope, key ]
|
445
|
+
else
|
446
|
+
if (@local_properties_cache[key]['system_properties']['string_only']) then
|
447
|
+
[ ope, key, @local_data_cache[key] ]
|
448
|
+
else
|
449
|
+
[ ope, key, @encode.call(@local_data_cache[key]) ]
|
450
|
+
end
|
451
|
+
end
|
452
|
+
} + \
|
453
|
+
@update_system_properties.map{|key, properties|
|
454
|
+
[ :system_properties, key, properties ]
|
455
|
+
} + \
|
456
|
+
@update_custom_properties.map{|key, properties|
|
457
|
+
[ :custom_properties, key, properties ]
|
458
|
+
}
|
459
|
+
end
|
460
|
+
private :write_list
|
461
|
+
|
462
|
+
def commit
|
463
|
+
write_list = write_list()
|
464
|
+
unless (write_list.empty?) then
|
465
|
+
@storage.write_and_commit(write_list)
|
466
|
+
for ope, key, value in write_list
|
467
|
+
case (ope)
|
468
|
+
when :write
|
469
|
+
@master_cache.delete(key)
|
470
|
+
@secondary_cache[key] = value
|
471
|
+
when :delete
|
472
|
+
@master_cache.delete(key)
|
473
|
+
@secondary_cache.delete(key)
|
474
|
+
end
|
475
|
+
@local_properties_cache.delete(key)
|
476
|
+
end
|
477
|
+
@update_system_properties.clear
|
478
|
+
@update_custom_properties.clear
|
479
|
+
@ope_map.clear
|
480
|
+
end
|
481
|
+
nil
|
482
|
+
end
|
483
|
+
|
484
|
+
def rollback
|
485
|
+
@local_data_cache.clear
|
486
|
+
@local_properties_cache.clear
|
487
|
+
@update_system_properties.clear
|
488
|
+
@update_custom_properties.clear
|
489
|
+
@ope_map.clear
|
490
|
+
nil
|
491
|
+
end
|
492
|
+
end
|
493
|
+
|
494
|
+
class ReadOnlyTransactionContext < TransactionContext
|
495
|
+
undef []=
|
496
|
+
undef update
|
497
|
+
undef delete
|
498
|
+
undef set_property
|
499
|
+
undef delete_property
|
500
|
+
undef delete_if
|
501
|
+
undef clear
|
502
|
+
undef commit
|
503
|
+
undef rollback
|
504
|
+
end
|
505
|
+
|
506
|
+
class ReadWriteTransactionContext < TransactionContext
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
# Local Variables:
|
511
|
+
# mode: Ruby
|
512
|
+
# indent-tabs-mode: nil
|
513
|
+
# End:
|