perobs 3.0.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +18 -17
- data/lib/perobs/BTree.rb +9 -44
- data/lib/perobs/BTreeNode.rb +116 -88
- data/lib/perobs/BTreeNodeCache.rb +10 -8
- data/lib/perobs/BTreeNodeLink.rb +1 -1
- data/lib/perobs/Cache.rb +14 -14
- data/lib/perobs/DynamoDB.rb +1 -1
- data/lib/perobs/EquiBlobsFile.rb +7 -2
- data/lib/perobs/FlatFile.rb +28 -49
- data/lib/perobs/FlatFileBlobHeader.rb +1 -19
- data/lib/perobs/FlatFileDB.rb +5 -0
- data/lib/perobs/LockFile.rb +3 -0
- data/lib/perobs/Object.rb +8 -3
- data/lib/perobs/ObjectBase.rb +6 -4
- data/lib/perobs/PersistentObjectCache.rb +153 -0
- data/lib/perobs/PersistentObjectCacheLine.rb +87 -0
- data/lib/perobs/SpaceTree.rb +5 -3
- data/lib/perobs/SpaceTreeNode.rb +15 -8
- data/lib/perobs/Store.rb +41 -13
- data/lib/perobs/version.rb +1 -1
- data/test/Array_spec.rb +38 -38
- data/test/BTree_spec.rb +45 -0
- data/test/EquiBlobsFile_spec.rb +0 -4
- data/test/FlatFileDB_spec.rb +1 -1
- data/test/Hash_spec.rb +14 -13
- data/test/Object_spec.rb +5 -5
- data/test/Store_spec.rb +62 -19
- data/test/perobs_spec.rb +7 -3
- metadata +4 -3
- data/lib/perobs/SpaceTreeNodeCache.rb +0 -149
@@ -31,11 +31,12 @@ module PEROBS
|
|
31
31
|
|
32
32
|
class BTreeNodeCache
|
33
33
|
|
34
|
-
def initialize
|
34
|
+
def initialize(tree)
|
35
|
+
@tree = tree
|
35
36
|
clear
|
36
37
|
end
|
37
38
|
|
38
|
-
def
|
39
|
+
def get(address)
|
39
40
|
if (node = @modified_nodes[address])
|
40
41
|
return node
|
41
42
|
end
|
@@ -48,7 +49,7 @@ module PEROBS
|
|
48
49
|
return node
|
49
50
|
end
|
50
51
|
|
51
|
-
|
52
|
+
BTreeNode::load(@tree, address)
|
52
53
|
end
|
53
54
|
|
54
55
|
def set_root(node)
|
@@ -58,12 +59,15 @@ module PEROBS
|
|
58
59
|
@top_nodes[node.node_address] = node
|
59
60
|
end
|
60
61
|
|
61
|
-
def insert(node)
|
62
|
+
def insert(node, modified = true)
|
62
63
|
unless node
|
63
64
|
PEROBS.log.fatal "nil cannot be cached"
|
64
65
|
end
|
65
66
|
node = node.get_node if node.is_a?(BTreeNodeLink)
|
66
67
|
|
68
|
+
if modified
|
69
|
+
@modified_nodes[node.node_address] = node
|
70
|
+
end
|
67
71
|
@ephemeral_nodes[node.node_address] = node
|
68
72
|
|
69
73
|
if !@top_nodes.include?(node) && node.is_top?
|
@@ -71,10 +75,8 @@ module PEROBS
|
|
71
75
|
end
|
72
76
|
end
|
73
77
|
|
74
|
-
def
|
75
|
-
|
76
|
-
@modified_nodes[node.node_address] = node
|
77
|
-
insert(node)
|
78
|
+
def _collect(address, ruby_object_id)
|
79
|
+
# Just a dummy for now
|
78
80
|
end
|
79
81
|
|
80
82
|
# Remove a node from the cache.
|
data/lib/perobs/BTreeNodeLink.rb
CHANGED
data/lib/perobs/Cache.rb
CHANGED
@@ -95,20 +95,20 @@ module PEROBS
|
|
95
95
|
|
96
96
|
# Return the PEROBS::Object with the specified ID or nil if not found.
|
97
97
|
# @param id [Integer] ID of the cached PEROBS::ObjectBase
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
98
|
+
def object_by_id(id)
|
99
|
+
idx = id & @mask
|
100
|
+
# The index is just a hash. We still need to check if the object IDs are
|
101
|
+
# actually the same before we can return the object.
|
102
|
+
if (obj = @writes[idx]) && obj._id == id
|
103
|
+
# The object was in the write cache.
|
104
|
+
return obj
|
105
|
+
elsif (obj = @reads[idx]) && obj._id == id
|
106
|
+
# The object was in the read cache.
|
107
|
+
return obj
|
108
|
+
end
|
109
|
+
|
110
|
+
nil
|
111
|
+
end
|
112
112
|
|
113
113
|
# Flush all pending writes to the persistant storage back-end.
|
114
114
|
def flush
|
data/lib/perobs/DynamoDB.rb
CHANGED
data/lib/perobs/EquiBlobsFile.rb
CHANGED
@@ -83,6 +83,7 @@ module PEROBS
|
|
83
83
|
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
84
84
|
PEROBS.log.fatal 'Database blob file is locked by another process'
|
85
85
|
end
|
86
|
+
@f.sync = true
|
86
87
|
end
|
87
88
|
|
88
89
|
# Close the blob file. This method must be called before the program is
|
@@ -92,6 +93,7 @@ module PEROBS
|
|
92
93
|
if @f
|
93
94
|
@f.flush
|
94
95
|
@f.flock(File::LOCK_UN)
|
96
|
+
@f.fsync
|
95
97
|
@f.close
|
96
98
|
@f = nil
|
97
99
|
end
|
@@ -103,7 +105,7 @@ module PEROBS
|
|
103
105
|
# Erase the backing store. This method should only be called when the file
|
104
106
|
# is not currently open.
|
105
107
|
def erase
|
106
|
-
|
108
|
+
@f = nil
|
107
109
|
File.delete(@file_name) if File.exist?(@file_name)
|
108
110
|
reset_counters
|
109
111
|
end
|
@@ -111,7 +113,10 @@ module PEROBS
|
|
111
113
|
# Flush out all unwritten data.
|
112
114
|
def sync
|
113
115
|
begin
|
114
|
-
|
116
|
+
if @f
|
117
|
+
@f.flush
|
118
|
+
@f.fsync
|
119
|
+
end
|
115
120
|
rescue IOError => e
|
116
121
|
PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
|
117
122
|
end
|
data/lib/perobs/FlatFile.rb
CHANGED
@@ -48,6 +48,7 @@ module PEROBS
|
|
48
48
|
@db_dir = dir
|
49
49
|
@f = nil
|
50
50
|
@index = BTree.new(@db_dir, 'index', INDEX_BTREE_ORDER)
|
51
|
+
@marks = BTree.new(@db_dir, 'marks', INDEX_BTREE_ORDER)
|
51
52
|
@space_list = SpaceTree.new(@db_dir)
|
52
53
|
end
|
53
54
|
|
@@ -71,6 +72,7 @@ module PEROBS
|
|
71
72
|
PEROBS.log.fatal "FlatFile database '#{file_name}' is locked by " +
|
72
73
|
"another process"
|
73
74
|
end
|
75
|
+
@f.sync = true
|
74
76
|
|
75
77
|
begin
|
76
78
|
@index.open(!new_db_created)
|
@@ -103,6 +105,7 @@ module PEROBS
|
|
103
105
|
if @f
|
104
106
|
@f.flush
|
105
107
|
@f.flock(File::LOCK_UN)
|
108
|
+
@f.fsync
|
106
109
|
@f.close
|
107
110
|
@f = nil
|
108
111
|
end
|
@@ -112,6 +115,7 @@ module PEROBS
|
|
112
115
|
def sync
|
113
116
|
begin
|
114
117
|
@f.flush
|
118
|
+
@f.fsync
|
115
119
|
rescue IOError => e
|
116
120
|
PEROBS.log.fatal "Cannot sync flat file database: #{e.message}"
|
117
121
|
end
|
@@ -148,7 +152,7 @@ module PEROBS
|
|
148
152
|
|
149
153
|
deleted_ids = []
|
150
154
|
each_blob_header do |pos, header|
|
151
|
-
if header.is_valid? &&
|
155
|
+
if header.is_valid? && @marks.get(header.id).nil?
|
152
156
|
delete_obj_by_address(pos, header.id)
|
153
157
|
deleted_ids << header.id
|
154
158
|
end
|
@@ -184,12 +188,12 @@ module PEROBS
|
|
184
188
|
# performance impact of compression is not compensated by writing
|
185
189
|
# less data to the storage.
|
186
190
|
compressed = false
|
187
|
-
if raw_obj.
|
191
|
+
if raw_obj.bytesize > 256
|
188
192
|
raw_obj = Zlib.deflate(raw_obj)
|
189
193
|
compressed = true
|
190
194
|
end
|
191
195
|
|
192
|
-
addr, length = find_free_blob(raw_obj.
|
196
|
+
addr, length = find_free_blob(raw_obj.bytesize)
|
193
197
|
begin
|
194
198
|
if length != -1
|
195
199
|
# Just a safeguard so we don't overwrite current data.
|
@@ -198,8 +202,8 @@ module PEROBS
|
|
198
202
|
PEROBS.log.fatal "Length in free list (#{length}) and header " +
|
199
203
|
"(#{header.length}) for address #{addr} don't match."
|
200
204
|
end
|
201
|
-
if raw_obj.
|
202
|
-
PEROBS.log.fatal "Object (#{raw_obj.
|
205
|
+
if raw_obj.bytesize > header.length
|
206
|
+
PEROBS.log.fatal "Object (#{raw_obj.bytesize}) is longer than " +
|
203
207
|
"blob space (#{header.length})."
|
204
208
|
end
|
205
209
|
if header.is_valid?
|
@@ -209,25 +213,19 @@ module PEROBS
|
|
209
213
|
end
|
210
214
|
flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
|
211
215
|
flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
|
212
|
-
|
213
|
-
# This method might be called in the middle of an operation that
|
214
|
-
# uses the mark flag. We must ensure that the flag is carried over
|
215
|
-
# to the new header.
|
216
|
-
flags |= (1 << FlatFileBlobHeader::MARK_FLAG_BIT)
|
217
|
-
end
|
218
|
-
FlatFileBlobHeader.new(@f, addr, flags, raw_obj.length, id, crc).write
|
216
|
+
FlatFileBlobHeader.new(@f, addr, flags, raw_obj.bytesize, id, crc).write
|
219
217
|
@f.write(raw_obj)
|
220
|
-
if length != -1 && raw_obj.
|
218
|
+
if length != -1 && raw_obj.bytesize < length
|
221
219
|
# The new object was not appended and it did not completely fill the
|
222
220
|
# free space. So we have to write a new header to mark the remaining
|
223
221
|
# empty space.
|
224
|
-
unless length - raw_obj.
|
222
|
+
unless length - raw_obj.bytesize >= FlatFileBlobHeader::LENGTH
|
225
223
|
PEROBS.log.fatal "Not enough space to append the empty space " +
|
226
|
-
"header (space: #{length} bytes, object: #{raw_obj.
|
224
|
+
"header (space: #{length} bytes, object: #{raw_obj.bytesize} " +
|
227
225
|
"bytes)."
|
228
226
|
end
|
229
227
|
space_address = @f.pos
|
230
|
-
space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.
|
228
|
+
space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.bytesize
|
231
229
|
FlatFileBlobHeader.new(@f, space_address, 0, space_length,
|
232
230
|
0, 0).write
|
233
231
|
# Register the new space with the space list.
|
@@ -272,6 +270,15 @@ module PEROBS
|
|
272
270
|
nil
|
273
271
|
end
|
274
272
|
|
273
|
+
def search_object(id)
|
274
|
+
each_blob_header do |pos, header|
|
275
|
+
return read_obj_by_address(pos, id)
|
276
|
+
end
|
277
|
+
|
278
|
+
nil
|
279
|
+
end
|
280
|
+
|
281
|
+
|
275
282
|
# Read the object at the specified address.
|
276
283
|
# @param addr [Integer] Offset in the flat file
|
277
284
|
# @param id [Integer] ID of the data blob
|
@@ -312,47 +319,19 @@ module PEROBS
|
|
312
319
|
# Mark the object with the given ID.
|
313
320
|
# @param id [Integer] ID of the object
|
314
321
|
def mark_obj_by_id(id)
|
315
|
-
|
316
|
-
mark_obj_by_address(addr, id)
|
317
|
-
end
|
318
|
-
end
|
319
|
-
|
320
|
-
# Mark the object at the specified address.
|
321
|
-
# @param addr [Integer] Offset in the file
|
322
|
-
# @param id [Integer] ID of the object
|
323
|
-
def mark_obj_by_address(addr, id)
|
324
|
-
FlatFileBlobHeader.read_at(@f, addr, id).set_mark_flag
|
322
|
+
@marks.insert(id, 0)
|
325
323
|
end
|
326
324
|
|
327
325
|
# Return true if the object with the given ID is marked, false otherwise.
|
328
326
|
# @param id [Integer] ID of the object
|
329
327
|
def is_marked_by_id?(id)
|
330
|
-
|
331
|
-
header = FlatFileBlobHeader.read_at(@f, addr, id)
|
332
|
-
return header.is_marked?
|
333
|
-
end
|
334
|
-
|
335
|
-
false
|
328
|
+
!@marks.get(id).nil?
|
336
329
|
end
|
337
330
|
|
338
331
|
# Clear alls marks.
|
339
332
|
def clear_all_marks
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
total_blob_count = 0
|
344
|
-
marked_blob_count = 0
|
345
|
-
|
346
|
-
each_blob_header do |pos, header|
|
347
|
-
total_blob_count += 1
|
348
|
-
if header.is_valid? && header.is_marked?
|
349
|
-
# Clear all valid and marked blocks.
|
350
|
-
marked_blob_count += 1
|
351
|
-
header.clear_mark_flag
|
352
|
-
end
|
353
|
-
end
|
354
|
-
PEROBS.log.info "#{marked_blob_count} marks in #{total_blob_count} " +
|
355
|
-
"objects cleared in #{Time.now - t} seconds"
|
333
|
+
@marks.erase
|
334
|
+
@marks.open
|
356
335
|
end
|
357
336
|
|
358
337
|
# Eliminate all the holes in the file. This is an in-place
|
@@ -464,7 +443,7 @@ module PEROBS
|
|
464
443
|
begin
|
465
444
|
@f.seek(pos + FlatFileBlobHeader::LENGTH)
|
466
445
|
buf = @f.read(header.length)
|
467
|
-
if buf.
|
446
|
+
if buf.bytesize != header.length
|
468
447
|
PEROBS.log.error "Premature end of file in blob with ID " +
|
469
448
|
"#{header.id}."
|
470
449
|
discard_damaged_blob(header) if repair
|
@@ -33,7 +33,7 @@ module PEROBS
|
|
33
33
|
#
|
34
34
|
# 1 Byte: Flags byte.
|
35
35
|
# Bit 0: 0 deleted entry, 1 valid entry
|
36
|
-
# Bit 1: 0
|
36
|
+
# Bit 1: 0 reserved, must be 0
|
37
37
|
# Bit 2: 0 uncompressed data, 1 compressed data
|
38
38
|
# Bit 3: 0 current entry, 1 outdated entry
|
39
39
|
# Bit 4 - 7: reserved, must be 0
|
@@ -50,7 +50,6 @@ module PEROBS
|
|
50
50
|
# The length of the header in bytes.
|
51
51
|
LENGTH = 21
|
52
52
|
VALID_FLAG_BIT = 0
|
53
|
-
MARK_FLAG_BIT = 1
|
54
53
|
COMPRESSED_FLAG_BIT = 2
|
55
54
|
OUTDATED_FLAG_BIT = 3
|
56
55
|
|
@@ -148,23 +147,6 @@ module PEROBS
|
|
148
147
|
bit_set?(VALID_FLAG_BIT)
|
149
148
|
end
|
150
149
|
|
151
|
-
# Return true if the blob has been marked.
|
152
|
-
def is_marked?
|
153
|
-
bit_set?(MARK_FLAG_BIT)
|
154
|
-
end
|
155
|
-
|
156
|
-
# Set the mark bit.
|
157
|
-
def set_mark_flag
|
158
|
-
set_flag(MARK_FLAG_BIT)
|
159
|
-
write_flags
|
160
|
-
end
|
161
|
-
|
162
|
-
# Clear the mark bit.
|
163
|
-
def clear_mark_flag
|
164
|
-
clear_flag(MARK_FLAG_BIT)
|
165
|
-
write_flags
|
166
|
-
end
|
167
|
-
|
168
150
|
# Return true if the blob contains compressed data.
|
169
151
|
def is_compressed?
|
170
152
|
bit_set?(COMPRESSED_FLAG_BIT)
|
data/lib/perobs/FlatFileDB.rb
CHANGED
@@ -87,6 +87,7 @@ module PEROBS
|
|
87
87
|
end
|
88
88
|
|
89
89
|
def FlatFileDB::delete_db(db_name)
|
90
|
+
close
|
90
91
|
FileUtils.rm_rf(db_name)
|
91
92
|
end
|
92
93
|
|
@@ -142,6 +143,10 @@ module PEROBS
|
|
142
143
|
end
|
143
144
|
end
|
144
145
|
|
146
|
+
def search_object(id)
|
147
|
+
@flat_file.search_object(id)
|
148
|
+
end
|
149
|
+
|
145
150
|
# This method must be called to initiate the marking process.
|
146
151
|
def clear_marks
|
147
152
|
@flat_file.clear_all_marks
|
data/lib/perobs/LockFile.rb
CHANGED
@@ -70,12 +70,14 @@ module PEROBS
|
|
70
70
|
while retries > 0
|
71
71
|
begin
|
72
72
|
@file = File.open(@file_name, File::RDWR | File::CREAT, 0644)
|
73
|
+
@file.sync = true
|
73
74
|
|
74
75
|
if @file.flock(File::LOCK_EX | File::LOCK_NB)
|
75
76
|
# We have taken the lock. Write the PID into the file and leave it
|
76
77
|
# open.
|
77
78
|
@file.write($$)
|
78
79
|
@file.flush
|
80
|
+
@file.fsync
|
79
81
|
@file.truncate(@file.pos)
|
80
82
|
PEROBS.log.debug "Lock file #{@file_name} has been taken for " +
|
81
83
|
"process #{$$}"
|
@@ -129,6 +131,7 @@ module PEROBS
|
|
129
131
|
|
130
132
|
begin
|
131
133
|
@file.flock(File::LOCK_UN)
|
134
|
+
@file.fsync
|
132
135
|
@file.close
|
133
136
|
forced_unlock
|
134
137
|
PEROBS.log.debug "Lock file #{@file_name} for PID #{$$} has been " +
|
data/lib/perobs/Object.rb
CHANGED
@@ -49,9 +49,11 @@ module PEROBS
|
|
49
49
|
attr_reader :attributes
|
50
50
|
|
51
51
|
# This method can be used to define instance variable for
|
52
|
-
# PEROBS::Object derived classes.
|
52
|
+
# PEROBS::Object derived classes. Persistent attributes always have
|
53
|
+
# getter and setter methods defined. So it's essentially equivalent to
|
54
|
+
# attr_accessor but additionally declares an attribute as persistent.
|
53
55
|
# @param attributes [Symbol] Name of the instance variable
|
54
|
-
def
|
56
|
+
def attr_persist(*attributes)
|
55
57
|
attributes.each do |attr_name|
|
56
58
|
unless attr_name.is_a?(Symbol)
|
57
59
|
PEROBS.log.fatal "name must be a symbol but is a " +
|
@@ -73,6 +75,9 @@ module PEROBS
|
|
73
75
|
end
|
74
76
|
end
|
75
77
|
|
78
|
+
# This is the deprecated name for the attr_persist method
|
79
|
+
alias po_attr attr_persist
|
80
|
+
|
76
81
|
end
|
77
82
|
|
78
83
|
attr_reader :attributes
|
@@ -108,7 +113,7 @@ module PEROBS
|
|
108
113
|
# object was saved with an earlier version of the program that did not yet
|
109
114
|
# have the instance variable. If you want to assign another PEROBS object
|
110
115
|
# to the variable you should use the block variant to avoid unnecessary
|
111
|
-
# creation of PEROBS
|
116
|
+
# creation of PEROBS objects that later need to be collected again.
|
112
117
|
def attr_init(attr, val = nil, &block)
|
113
118
|
if _all_attributes.include?(attr)
|
114
119
|
unless instance_variable_defined?('@' + attr.to_s)
|
data/lib/perobs/ObjectBase.rb
CHANGED
@@ -133,7 +133,8 @@ module PEROBS
|
|
133
133
|
@store = p.store
|
134
134
|
@_id = p.id
|
135
135
|
@store._register_in_memory(self, @_id)
|
136
|
-
ObjectSpace.define_finalizer(
|
136
|
+
ObjectSpace.define_finalizer(
|
137
|
+
self, ObjectBase._finalize(@store, @_id, object_id))
|
137
138
|
@_stash_map = nil
|
138
139
|
# Allocate a proxy object for this object. User code should only operate
|
139
140
|
# on this proxy, never on self.
|
@@ -144,8 +145,8 @@ module PEROBS
|
|
144
145
|
# is done this way to prevent the Proc object hanging on to a reference to
|
145
146
|
# self which would prevent the object from being collected. This internal
|
146
147
|
# method is not intended for users to call.
|
147
|
-
def ObjectBase._finalize(store, id)
|
148
|
-
proc { store._collect(id) }
|
148
|
+
def ObjectBase._finalize(store, id, ruby_object_id)
|
149
|
+
proc { store._collect(id, ruby_object_id) }
|
149
150
|
end
|
150
151
|
|
151
152
|
# Library internal method to transfer the Object to a new store.
|
@@ -158,7 +159,8 @@ module PEROBS
|
|
158
159
|
# Register the object as in-memory object with the new store.
|
159
160
|
@store._register_in_memory(self, @_id)
|
160
161
|
# Register the finalizer for the new store.
|
161
|
-
ObjectSpace.define_finalizer(
|
162
|
+
ObjectSpace.define_finalizer(
|
163
|
+
self, ObjectBase._finalize(@store, @_id, object_id))
|
162
164
|
@myself = POXReference.new(@store, @_id)
|
163
165
|
end
|
164
166
|
|