perobs 3.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/perobs/Array.rb +3 -3
- data/lib/perobs/BTreeBlob.rb +10 -10
- data/lib/perobs/BTreeDB.rb +6 -6
- data/lib/perobs/Cache.rb +2 -2
- data/lib/perobs/ClassMap.rb +2 -2
- data/lib/perobs/DynamoDB.rb +5 -5
- data/lib/perobs/EquiBlobsFile.rb +14 -7
- data/lib/perobs/FlatFile.rb +29 -7
- data/lib/perobs/FlatFileBlobHeader.rb +7 -12
- data/lib/perobs/FlatFileDB.rb +6 -6
- data/lib/perobs/Hash.rb +2 -2
- data/lib/perobs/Object.rb +2 -2
- data/lib/perobs/ObjectBase.rb +1 -1
- data/lib/perobs/SpaceTree.rb +47 -42
- data/lib/perobs/SpaceTreeNode.rb +89 -69
- data/lib/perobs/SpaceTreeNodeCache.rb +84 -11
- data/lib/perobs/SpaceTreeNodeLink.rb +1 -1
- data/lib/perobs/StackFile.rb +1 -1
- data/lib/perobs/Store.rb +14 -14
- data/lib/perobs/TreeDB.rb +7 -7
- data/lib/perobs/version.rb +1 -1
- data/test/Array_spec.rb +4 -4
- data/test/BTreeDB_spec.rb +2 -2
- data/test/SpaceTree_spec.rb +23 -8
- data/test/Store_spec.rb +3 -1
- metadata +2 -2
@@ -34,40 +34,113 @@ module PEROBS
|
|
34
34
|
# Simple cache that can hold up to size SpaceTreeNode entries. Entries are
|
35
35
|
# hashed with a simple node_address % size function. This keeps the
|
36
36
|
# overhead for managing the cache extremely low yet giving an OK
|
37
|
-
# probability to have cache hits.
|
37
|
+
# probability to have cache hits. The cache also keeps track if a node is
|
38
|
+
# still in memory or needs to be reloaded from the file. All node accesses
|
39
|
+
# must always go through this cache to avoid having duplicate in-memory
|
40
|
+
# nodes for the same on-disk node.
|
38
41
|
# @param size [Integer] maximum number of cache entries
|
39
|
-
def initialize(size)
|
42
|
+
def initialize(tree, size)
|
43
|
+
@tree = tree
|
40
44
|
@size = size
|
41
45
|
clear
|
42
46
|
end
|
43
47
|
|
44
|
-
# Insert
|
45
|
-
# @param node [SpaceTreeNode]
|
46
|
-
def
|
47
|
-
@
|
48
|
+
# Insert an unmodified node into the cache.
|
49
|
+
# @param node [SpaceTreeNode] Unmodified SpaceTreeNode
|
50
|
+
def insert_unmodified(node)
|
51
|
+
@in_memory_nodes[node.node_address] = node.object_id
|
52
|
+
@unmodified_nodess[node.node_address % @size] = node
|
53
|
+
end
|
54
|
+
|
55
|
+
# Insert a modified node into the cache.
|
56
|
+
# @param node [SpaceTreeNode] Modified SpaceTreeNode
|
57
|
+
# @param address [Integer] Address of the node in the file
|
58
|
+
def insert_modified(node)
|
59
|
+
@in_memory_nodes[node.node_address] = node.object_id
|
60
|
+
index = node.node_address % @size
|
61
|
+
if (old_node = @modified_nodes[index])
|
62
|
+
# If the object is already in the modified object list, we don't have
|
63
|
+
# to do anything.
|
64
|
+
return if old_node.node_address == node.node_address
|
65
|
+
# If the new object will replace an existing entry in the cash we have
|
66
|
+
# to save the object first.
|
67
|
+
old_node.save
|
68
|
+
end
|
69
|
+
|
70
|
+
@modified_nodes[index] = node
|
48
71
|
end
|
49
72
|
|
50
73
|
# Retrieve a node reference from the cache.
|
51
74
|
# @param address [Integer] address of the node to retrieve.
|
52
75
|
def get(address)
|
53
|
-
node = @
|
76
|
+
node = @unmodified_nodess[address % @size]
|
54
77
|
# We can have collisions. Check if the cached node really matches the
|
55
78
|
# requested address.
|
56
|
-
|
79
|
+
return node if node && node.node_address == address
|
80
|
+
|
81
|
+
node = @modified_nodes[address % @size]
|
82
|
+
# We can have collisions. Check if the cached node really matches the
|
83
|
+
# requested address.
|
84
|
+
return node if node && node.node_address == address
|
85
|
+
|
86
|
+
if (obj_id = @in_memory_nodes[address])
|
87
|
+
# We have the node in memory so we can just return it.
|
88
|
+
begin
|
89
|
+
node = ObjectSpace._id2ref(obj_id)
|
90
|
+
unless node.node_address == address
|
91
|
+
raise RuntimeError, "In memory list is corrupted"
|
92
|
+
end
|
93
|
+
insert_unmodified(node)
|
94
|
+
return node
|
95
|
+
rescue RangeError
|
96
|
+
# Due to a race condition the object can still be in the
|
97
|
+
# @in_memory_nodes list but has been collected already by the Ruby
|
98
|
+
# GC. In that case we need to load it again.
|
99
|
+
@in_memory_nodes.delete(address)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
SpaceTreeNode::load(@tree, address)
|
57
104
|
end
|
58
105
|
|
59
106
|
# Remove a node from the cache.
|
60
107
|
# @param address [Integer] address of node to remove.
|
61
108
|
def delete(address)
|
109
|
+
# The object is likely still in memory, but we really don't want to
|
110
|
+
# access it anymore.
|
111
|
+
@in_memory_nodes.delete(address)
|
112
|
+
|
62
113
|
index = address % @size
|
63
|
-
if (node = @
|
64
|
-
@
|
114
|
+
if (node = @unmodified_nodess[index]) && node.node_address == address
|
115
|
+
@unmodified_nodess[index] = nil
|
116
|
+
end
|
117
|
+
|
118
|
+
if (node = @modified_nodes[index]) && node.node_address == address
|
119
|
+
@modified_nodes[index] = nil
|
65
120
|
end
|
66
121
|
end
|
67
122
|
|
123
|
+
# Remove a node from the in-memory list. This is an internal method
|
124
|
+
# and should never be called from user code. It will be called from a
|
125
|
+
# finalizer, so many restrictions apply!
|
126
|
+
# @param node_address [Integer] Node address of the node to remove from
|
127
|
+
# the list
|
128
|
+
def _collect(address)
|
129
|
+
@in_memory_nodes.delete(address)
|
130
|
+
end
|
131
|
+
|
132
|
+
# Write all modified objects into the backing store.
|
133
|
+
def flush
|
134
|
+
@modified_nodes.each { |node| node.save if node }
|
135
|
+
@modified_nodes = ::Array.new(@size)
|
136
|
+
end
|
137
|
+
|
68
138
|
# Remove all entries from the cache.
|
69
139
|
def clear
|
70
|
-
|
140
|
+
# A hash that stores all objects by ID that are currently in memory.
|
141
|
+
@in_memory_nodes = {}
|
142
|
+
@unmodified_nodess = ::Array.new(@size)
|
143
|
+
@modified_nodes = ::Array.new(@size)
|
71
144
|
end
|
72
145
|
|
73
146
|
end
|
data/lib/perobs/StackFile.rb
CHANGED
@@ -36,7 +36,7 @@ module PEROBS
|
|
36
36
|
# Create a new stack file in the given directory with the given file name.
|
37
37
|
# @param dir [String] Directory
|
38
38
|
# @param name [String] File name
|
39
|
-
# @param entry_bytes [
|
39
|
+
# @param entry_bytes [Integer] Number of bytes each entry must have
|
40
40
|
def initialize(dir, name, entry_bytes)
|
41
41
|
@file_name = File.join(dir, name + '.stack')
|
42
42
|
@entry_bytes = entry_bytes
|
data/lib/perobs/Store.rb
CHANGED
@@ -26,7 +26,6 @@
|
|
26
26
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
27
|
|
28
28
|
require 'set'
|
29
|
-
require 'weakref'
|
30
29
|
|
31
30
|
require 'perobs/Log'
|
32
31
|
require 'perobs/Handle'
|
@@ -120,7 +119,7 @@ module PEROBS
|
|
120
119
|
# :json : About half as fast as marshal, but the
|
121
120
|
# format is rock solid and portable between
|
122
121
|
# languages. It only supports basic Ruby data types
|
123
|
-
# like String,
|
122
|
+
# like String, Integer, Float, Array, Hash. This is
|
124
123
|
# the default option.
|
125
124
|
# :yaml : Can also handle most Ruby data types and is
|
126
125
|
# portable between Ruby versions (1.9 and later).
|
@@ -221,7 +220,7 @@ module PEROBS
|
|
221
220
|
# For library internal use only!
|
222
221
|
# This method will create a new PEROBS object.
|
223
222
|
# @param klass [BasicObject] Class of the object to create
|
224
|
-
# @param id [
|
223
|
+
# @param id [Integer] Requested object ID
|
225
224
|
# @param args [Array] Arguments to pass to the object constructor.
|
226
225
|
# @return [BasicObject] Newly constructed PEROBS object
|
227
226
|
def _construct_po(klass, id, *args)
|
@@ -297,7 +296,7 @@ module PEROBS
|
|
297
296
|
# from the back-end storage. The garbage collector is not invoked
|
298
297
|
# automatically. Depending on your usage pattern, you need to call this
|
299
298
|
# method periodically.
|
300
|
-
# @return [
|
299
|
+
# @return [Integer] The number of collected objects
|
301
300
|
def gc
|
302
301
|
if @cache.in_transaction?
|
303
302
|
PEROBS.log.fatal 'You cannot call gc() during a transaction'
|
@@ -314,11 +313,12 @@ module PEROBS
|
|
314
313
|
if (obj = @in_memory_objects[id])
|
315
314
|
# We have the object in memory so we can just return it.
|
316
315
|
begin
|
317
|
-
return obj
|
318
|
-
rescue
|
316
|
+
return ObjectSpace._id2ref(obj)
|
317
|
+
rescue RangeError
|
319
318
|
# Due to a race condition the object can still be in the
|
320
319
|
# @in_memory_objects list but has been collected already by the Ruby
|
321
320
|
# GC. In that case we need to load it again.
|
321
|
+
@in_memory_objects.delete(id)
|
322
322
|
end
|
323
323
|
end
|
324
324
|
|
@@ -342,7 +342,7 @@ module PEROBS
|
|
342
342
|
# unreadable object is found, the reference will simply be deleted.
|
343
343
|
# @param repair [TrueClass/FalseClass] true if a repair attempt should be
|
344
344
|
# made.
|
345
|
-
# @return [
|
345
|
+
# @return [Integer] The number of references to bad objects found.
|
346
346
|
def check(repair = false)
|
347
347
|
# All objects must have in-db version.
|
348
348
|
sync
|
@@ -436,7 +436,7 @@ module PEROBS
|
|
436
436
|
# Internal method. Don't use this outside of this library!
|
437
437
|
# Generate a new unique ID that is not used by any other object. It uses
|
438
438
|
# random numbers between 0 and 2**64 - 1.
|
439
|
-
# @return [
|
439
|
+
# @return [Integer]
|
440
440
|
def _new_id
|
441
441
|
begin
|
442
442
|
# Generate a random number. It's recommended to not store more than
|
@@ -454,15 +454,15 @@ module PEROBS
|
|
454
454
|
# happens the object finalizer is triggered and calls _forget() to
|
455
455
|
# remove the object from this hash again.
|
456
456
|
# @param obj [BasicObject] Object to register
|
457
|
-
# @param id [
|
457
|
+
# @param id [Integer] object ID
|
458
458
|
def _register_in_memory(obj, id)
|
459
|
-
@in_memory_objects[id] =
|
459
|
+
@in_memory_objects[id] = obj.object_id
|
460
460
|
end
|
461
461
|
|
462
462
|
# Remove the object from the in-memory list. This is an internal method
|
463
463
|
# and should never be called from user code. It will be called from a
|
464
464
|
# finalizer, so many restrictions apply!
|
465
|
-
# @param id [
|
465
|
+
# @param id [Integer] Object ID of object to remove from the list
|
466
466
|
def _collect(id, ignore_errors = false)
|
467
467
|
@in_memory_objects.delete(id)
|
468
468
|
end
|
@@ -502,10 +502,10 @@ module PEROBS
|
|
502
502
|
|
503
503
|
# Check the object with the given start_id and all other objects that are
|
504
504
|
# somehow reachable from the start object.
|
505
|
-
# @param start_id [
|
505
|
+
# @param start_id [Integer] ID of the top-level object to start
|
506
506
|
# with
|
507
507
|
# @param repair [Boolean] Delete refernces to broken objects if true
|
508
|
-
# @return [
|
508
|
+
# @return [Integer] The number of references to bad objects.
|
509
509
|
def check_object(start_id, repair)
|
510
510
|
errors = 0
|
511
511
|
@db.mark(start_id)
|
@@ -518,7 +518,7 @@ module PEROBS
|
|
518
518
|
# Get the next PEROBS object to check
|
519
519
|
ref_obj, id = todo_list.pop
|
520
520
|
|
521
|
-
if (obj = object_by_id(id))
|
521
|
+
if (obj = object_by_id(id))
|
522
522
|
# The object exists and is OK. Mark is as checked.
|
523
523
|
@db.mark(id)
|
524
524
|
# Now look at all other objects referenced by this object.
|
data/lib/perobs/TreeDB.rb
CHANGED
@@ -103,7 +103,7 @@ module PEROBS
|
|
103
103
|
end
|
104
104
|
|
105
105
|
# Return true if the object with given ID exists
|
106
|
-
# @param id [
|
106
|
+
# @param id [Integer]
|
107
107
|
def include?(id)
|
108
108
|
!(blob = find_blob(id)).nil? && !blob.find(id).nil?
|
109
109
|
end
|
@@ -143,7 +143,7 @@ module PEROBS
|
|
143
143
|
end
|
144
144
|
|
145
145
|
# Load the given object from the filesystem.
|
146
|
-
# @param id [
|
146
|
+
# @param id [Integer] object ID
|
147
147
|
# @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
|
148
148
|
# not exist
|
149
149
|
def get_object(id)
|
@@ -166,13 +166,13 @@ module PEROBS
|
|
166
166
|
end
|
167
167
|
|
168
168
|
# Mark an object.
|
169
|
-
# @param id [
|
169
|
+
# @param id [Integer] ID of the object to mark
|
170
170
|
def mark(id)
|
171
171
|
(blob = find_blob(id)) && blob.mark(id)
|
172
172
|
end
|
173
173
|
|
174
174
|
# Check if the object is marked.
|
175
|
-
# @param id [
|
175
|
+
# @param id [Integer] ID of the object to check
|
176
176
|
# @param ignore_errors [Boolean] If set to true no errors will be raised
|
177
177
|
# for non-existing objects.
|
178
178
|
def is_marked?(id, ignore_errors = false)
|
@@ -187,7 +187,7 @@ module PEROBS
|
|
187
187
|
end
|
188
188
|
|
189
189
|
# Check if the stored object is syntactically correct.
|
190
|
-
# @param id [
|
190
|
+
# @param id [Integer] Object ID
|
191
191
|
# @param repair [TrueClass/FalseClass] True if an repair attempt should be
|
192
192
|
# made.
|
193
193
|
# @return [TrueClass/FalseClass] True if the object is OK, otherwise
|
@@ -206,7 +206,7 @@ module PEROBS
|
|
206
206
|
# Store the given serialized object into the cluster files. This method is
|
207
207
|
# for internal use only!
|
208
208
|
# @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
|
209
|
-
# @param id [
|
209
|
+
# @param id [Integer] Object ID
|
210
210
|
def put_raw_object(raw, id)
|
211
211
|
find_blob(id, true).write_object(id, raw)
|
212
212
|
end
|
@@ -226,7 +226,7 @@ module PEROBS
|
|
226
226
|
Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
|
227
227
|
# Extract the 01-part of the filename
|
228
228
|
lsb_string = File.basename(fqfn)[0..-6]
|
229
|
-
# Convert the lsb_string into a
|
229
|
+
# Convert the lsb_string into a Integer
|
230
230
|
lsb = Integer('0b' + lsb_string)
|
231
231
|
# Bit mask to match the LSBs
|
232
232
|
mask = (2 ** lsb_string.length) - 1
|
data/lib/perobs/version.rb
CHANGED
data/test/Array_spec.rb
CHANGED
@@ -169,8 +169,8 @@ describe PEROBS::Array do
|
|
169
169
|
|
170
170
|
it 'should support collect!()' do
|
171
171
|
a = cpa([ 1, 'cat', 1..1 ])
|
172
|
-
expect(a.collect! { |e| e.class }).to eq([
|
173
|
-
pcheck { expect(a).to eq([
|
172
|
+
expect(a.collect! { |e| e.class }).to eq([ Integer, String, Range ])
|
173
|
+
pcheck { expect(a).to eq([ Integer, String, Range ]) }
|
174
174
|
|
175
175
|
a = cpa([ 1, 'cat', 1..1 ])
|
176
176
|
expect(a.collect! { 99 }).to eq([ 99, 99, 99])
|
@@ -179,8 +179,8 @@ describe PEROBS::Array do
|
|
179
179
|
|
180
180
|
it 'should support map!()' do
|
181
181
|
a = cpa([ 1, 'cat', 1..1 ])
|
182
|
-
expect(a.map! { |e| e.class }).to eq([
|
183
|
-
pcheck { expect(a).to eq([
|
182
|
+
expect(a.map! { |e| e.class }).to eq([ Integer, String, Range ])
|
183
|
+
pcheck { expect(a).to eq([ Integer, String, Range ]) }
|
184
184
|
|
185
185
|
a = cpa([ 1, 'cat', 1..1 ])
|
186
186
|
expect(a.map! { 99 }).to eq([ 99, 99, 99])
|
data/test/BTreeDB_spec.rb
CHANGED
@@ -56,7 +56,7 @@ describe PEROBS::BTreeDB do
|
|
56
56
|
expect(@db.include?(0)).to be false
|
57
57
|
h = {
|
58
58
|
'String' => 'What god has wrought',
|
59
|
-
'
|
59
|
+
'Integer' => 42,
|
60
60
|
'Float' => 3.14,
|
61
61
|
'True' => true,
|
62
62
|
'False' => false,
|
@@ -75,7 +75,7 @@ describe PEROBS::BTreeDB do
|
|
75
75
|
expect(@db.include?(0)).to be false
|
76
76
|
h = {
|
77
77
|
'String' => 'What god has wrought',
|
78
|
-
'
|
78
|
+
'Integer' => 42,
|
79
79
|
'Float' => 3.14,
|
80
80
|
'True' => true,
|
81
81
|
'False' => false,
|
data/test/SpaceTree_spec.rb
CHANGED
@@ -94,12 +94,12 @@ EOT
|
|
94
94
|
end
|
95
95
|
|
96
96
|
it 'should find the smallest node' do
|
97
|
-
node = @m.
|
97
|
+
node = @m.root.find_smallest_node
|
98
98
|
expect(node.size).to eql(2)
|
99
99
|
end
|
100
100
|
|
101
101
|
it 'should find the largest node' do
|
102
|
-
node = @m.
|
102
|
+
node = @m.root.find_largest_node
|
103
103
|
expect(node.size).to eql(32)
|
104
104
|
expect(node.node_address).to eql(5)
|
105
105
|
expect(node.parent.size).to eql(16)
|
@@ -119,12 +119,14 @@ EOT
|
|
119
119
|
add_sizes([ 10, 5, 15, 10 ])
|
120
120
|
expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 10], [2, 15]])
|
121
121
|
expect(@m.get_space(10)).to eql([0, 10])
|
122
|
+
expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
|
122
123
|
|
123
124
|
@m.clear
|
124
125
|
add_sizes([ 10, 5, 15, 10, 10 ])
|
125
126
|
expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 10], [3, 10], [2, 15]])
|
126
127
|
expect(@m.get_space(10)).to eql([0, 10])
|
127
128
|
expect(@m.get_space(10)).to eql([4, 10])
|
129
|
+
expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
|
128
130
|
end
|
129
131
|
|
130
132
|
it 'should delete a smaller node' do
|
@@ -208,14 +210,15 @@ EOT
|
|
208
210
|
it 'should support a real-world traffic pattern' do
|
209
211
|
address = 0
|
210
212
|
spaces = []
|
211
|
-
|
212
|
-
|
213
|
+
@m.clear
|
214
|
+
0.upto(1000) do
|
215
|
+
case rand(4)
|
213
216
|
when 0
|
214
217
|
# Insert new values
|
215
218
|
rand(9).times do
|
216
219
|
size = 20 + rand(5000)
|
217
220
|
@m.add_space(address, size)
|
218
|
-
spaces << [address, size]
|
221
|
+
spaces << [ address, size ]
|
219
222
|
address += size
|
220
223
|
end
|
221
224
|
when 1
|
@@ -228,9 +231,21 @@ EOT
|
|
228
231
|
end
|
229
232
|
end
|
230
233
|
when 2
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
+
if rand(10) == 0
|
235
|
+
expect(@m.check).to be true
|
236
|
+
spaces.each do |address, size|
|
237
|
+
expect(@m.has_space?(address, size)).to be true
|
238
|
+
end
|
239
|
+
@m.each do |address, size|
|
240
|
+
expect(spaces.include?([ address, size ])).to be true
|
241
|
+
end
|
242
|
+
end
|
243
|
+
when 3
|
244
|
+
if rand(100) == 0
|
245
|
+
expect(@m.check).to be true
|
246
|
+
@m.close
|
247
|
+
@m.open
|
248
|
+
expect(@m.check).to be true
|
234
249
|
end
|
235
250
|
end
|
236
251
|
end
|
data/test/Store_spec.rb
CHANGED
@@ -422,7 +422,9 @@ describe PEROBS::Store do
|
|
422
422
|
@store.sync
|
423
423
|
GC.start
|
424
424
|
# Now the Person should be gone from memory.
|
425
|
-
|
425
|
+
# Ruby 2.3 and later has changed the GC so that this does not work
|
426
|
+
# reliably anymore. The GC seems to operate lazyly.
|
427
|
+
#expect(@store.statistics[:in_memory_objects]).to eq(1)
|
426
428
|
end
|
427
429
|
|
428
430
|
it 'should handle nested constructors' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|