perobs 4.3.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +26 -15
- data/lib/perobs/BTreeNode.rb +14 -14
- data/lib/perobs/BigArrayNode.rb +1 -1
- data/lib/perobs/Cache.rb +19 -6
- data/lib/perobs/EquiBlobsFile.rb +2 -0
- data/lib/perobs/FlatFile.rb +3 -4
- data/lib/perobs/IDListPageRecord.rb +1 -1
- data/lib/perobs/Store.rb +161 -124
- data/lib/perobs/version.rb +1 -1
- data/perobs.gemspec +1 -1
- metadata +5 -6
- data/lib/perobs/BTreeNodeCache.rb +0 -109
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2c526c97aab15c09f8e8acbcb432203e7d60fef8c504df6ddb0b8f35459cca0
|
4
|
+
data.tar.gz: ad077bb879c041289c7a474e87645064ff84203c674640aff59b5f23923ac1b7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b86fc662ef8bbf623ee7da777f023a74e3db9502df18c45dcda9ce244982225aaec7e74ce25378147c807775765c57d801810ac16be01d2145227bb0642caa3b
|
7
|
+
data.tar.gz: d0c5a79eb60a1221bc385a1fafbc77a1aaa1e47782b3ea27ca92f5f7c539804bc00a388fdddc1ab956d7805dbdd52d055f5cdd76a8bb155ec5ada4335b7b9175
|
data/README.md
CHANGED
@@ -6,11 +6,12 @@ them from PEROBS::Object. They will be in memory when needed and
|
|
6
6
|
transparently stored into a persistent storage.
|
7
7
|
|
8
8
|
This library is ideal for Ruby applications that work on huge, mostly
|
9
|
-
|
9
|
+
static data sets and usually process a small subset of the data at a
|
10
10
|
time. To ensure data consistency of a larger data set, you can use
|
11
11
|
transactions to make modifications of multiple objects atomicaly.
|
12
12
|
Transactions can be nested and are aborted when an exception is
|
13
|
-
raised.
|
13
|
+
raised. PEROBS is thread-safe, so you can use it in a multi-threaded
|
14
|
+
application.
|
14
15
|
|
15
16
|
## Usage
|
16
17
|
|
@@ -120,15 +121,18 @@ class Person < PEROBS::Object
|
|
120
121
|
|
121
122
|
end
|
122
123
|
|
123
|
-
|
124
|
-
store
|
125
|
-
store['
|
126
|
-
|
127
|
-
jim
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
124
|
+
begin
|
125
|
+
store = PEROBS::Store.new('family')
|
126
|
+
store['grandpa'] = joe = store.new(Person, 'Joe')
|
127
|
+
store['grandma'] = jane = store.new(Person, 'Jane')
|
128
|
+
jim = store.new(Person, 'Jim')
|
129
|
+
jim.father = joe
|
130
|
+
joe.kids << jim
|
131
|
+
jim.mother = jane
|
132
|
+
jane.kids << jim
|
133
|
+
ensure
|
134
|
+
store.exit
|
135
|
+
end
|
132
136
|
```
|
133
137
|
|
134
138
|
When you run this script, a folder named 'family' will be created. It
|
@@ -166,9 +170,15 @@ object to another object.
|
|
166
170
|
|
167
171
|
### Caveats and known issues
|
168
172
|
|
169
|
-
|
170
|
-
|
171
|
-
|
173
|
+
You cannot simultaneously access the database from multiple
|
174
|
+
applications concurrently. The library uses locks to ensure that only
|
175
|
+
one Store object is accessing the database at a time.
|
176
|
+
|
177
|
+
In case the application terminates without calling Store::exit(), the
|
178
|
+
database or the database index could get corrupted. To check the
|
179
|
+
consistency of your database you can use Store::check(). To check and
|
180
|
+
repair the database you can call Store::repair(). Depending on the
|
181
|
+
size of your database, these operations can last minutes to hours.
|
172
182
|
|
173
183
|
## Installation
|
174
184
|
|
@@ -188,7 +198,8 @@ Or install it yourself as:
|
|
188
198
|
|
189
199
|
## Copyright and License
|
190
200
|
|
191
|
-
Copyright (c) 2015, 2016, 2017
|
201
|
+
Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 by
|
202
|
+
Chris Schlaeger <chris@taskjuggler.org>
|
192
203
|
|
193
204
|
PEROBS and all accompanying files are licensed under this MIT License
|
194
205
|
|
data/lib/perobs/BTreeNode.rb
CHANGED
@@ -78,7 +78,7 @@ module PEROBS
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
-
# Create a new
|
81
|
+
# Create a new BTreeNode. This method should be used for the creation
|
82
82
|
# of new nodes instead of calling the constructor directly.
|
83
83
|
# @param tree [BTree] The tree the new node should belong to
|
84
84
|
# @param parent [BTreeNode] The parent node
|
@@ -130,7 +130,8 @@ module PEROBS
|
|
130
130
|
ary = bytes.unpack(BTreeNode::node_bytes_format(tree))
|
131
131
|
# Read is_leaf
|
132
132
|
if ary[0] != 0 && ary[0] != 1
|
133
|
-
PEROBS.log.fatal "First byte of a BTreeNode entry
|
133
|
+
PEROBS.log.fatal "First byte of a BTreeNode entry at address " +
|
134
|
+
"#{address} must be 0 or 1 but is #{ary[0]}"
|
134
135
|
end
|
135
136
|
is_leaf = ary[0] == 0 ? false : true
|
136
137
|
# This is the number of keys this node has.
|
@@ -350,6 +351,7 @@ module PEROBS
|
|
350
351
|
unless @parent
|
351
352
|
# The node is the root node. We need to create a parent node first.
|
352
353
|
self.parent = link(BTreeNode::create(@tree, nil, false))
|
354
|
+
@tree.node_cache.insert(self)
|
353
355
|
@parent.set_child(0, self)
|
354
356
|
@tree.set_root(@parent)
|
355
357
|
end
|
@@ -382,6 +384,7 @@ module PEROBS
|
|
382
384
|
end
|
383
385
|
|
384
386
|
i = search_key_index(key)
|
387
|
+
@tree.node_cache.insert(self)
|
385
388
|
if @keys[i] == key
|
386
389
|
# Overwrite existing entries
|
387
390
|
@keys[i] = key
|
@@ -390,7 +393,6 @@ module PEROBS
|
|
390
393
|
else
|
391
394
|
@children[i + 1] = link(value_or_child)
|
392
395
|
end
|
393
|
-
@tree.node_cache.insert(self)
|
394
396
|
|
395
397
|
return false
|
396
398
|
else
|
@@ -401,7 +403,6 @@ module PEROBS
|
|
401
403
|
else
|
402
404
|
@children.insert(i + 1, link(value_or_child))
|
403
405
|
end
|
404
|
-
@tree.node_cache.insert(self)
|
405
406
|
|
406
407
|
return true
|
407
408
|
end
|
@@ -417,8 +418,8 @@ module PEROBS
|
|
417
418
|
update_branch_key(key) if index == 0
|
418
419
|
|
419
420
|
# Delete the corresponding value.
|
420
|
-
removed_value = @values.delete_at(index)
|
421
421
|
@tree.node_cache.insert(self)
|
422
|
+
removed_value = @values.delete_at(index)
|
422
423
|
|
423
424
|
if @keys.length < min_keys
|
424
425
|
if @prev_sibling && @prev_sibling.parent == @parent
|
@@ -481,9 +482,9 @@ module PEROBS
|
|
481
482
|
PEROBS.log.fatal "Leaf nodes are too big to merge"
|
482
483
|
end
|
483
484
|
|
485
|
+
@tree.node_cache.insert(self)
|
484
486
|
@keys += node.keys
|
485
487
|
@values += node.values
|
486
|
-
@tree.node_cache.insert(self)
|
487
488
|
|
488
489
|
node.parent.remove_child(node)
|
489
490
|
end
|
@@ -494,11 +495,11 @@ module PEROBS
|
|
494
495
|
end
|
495
496
|
|
496
497
|
index = @parent.search_node_index(node) - 1
|
498
|
+
@tree.node_cache.insert(self)
|
497
499
|
@keys << @parent.keys[index]
|
498
500
|
@keys += node.keys
|
499
501
|
node.children.each { |c| c.parent = link(self) }
|
500
502
|
@children += node.children
|
501
|
-
@tree.node_cache.insert(self)
|
502
503
|
|
503
504
|
node.parent.remove_child(node)
|
504
505
|
end
|
@@ -525,6 +526,7 @@ module PEROBS
|
|
525
526
|
"#{dest_node.is_leaf} node must be of same kind"
|
526
527
|
end
|
527
528
|
|
529
|
+
@tree.node_cache.insert(dest_node)
|
528
530
|
dest_node.keys[dst_idx, count] = @keys[src_idx, count]
|
529
531
|
if @is_leaf
|
530
532
|
# For leaves we copy the keys and corresponding values.
|
@@ -537,17 +539,17 @@ module PEROBS
|
|
537
539
|
dest_node.set_child(dst_idx + i, @children[src_idx + i])
|
538
540
|
end
|
539
541
|
end
|
540
|
-
@tree.node_cache.insert(dest_node)
|
541
542
|
end
|
542
543
|
|
543
544
|
def parent=(p)
|
544
|
-
@parent = p
|
545
545
|
@tree.node_cache.insert(self)
|
546
|
+
@parent = p
|
546
547
|
|
547
548
|
p
|
548
549
|
end
|
549
550
|
|
550
551
|
def prev_sibling=(node)
|
552
|
+
@tree.node_cache.insert(self)
|
551
553
|
@prev_sibling = node
|
552
554
|
if node.nil? && @is_leaf
|
553
555
|
# If this node is a leaf node without a previous sibling we need to
|
@@ -555,14 +557,12 @@ module PEROBS
|
|
555
557
|
@tree.set_first_leaf(BTreeNodeLink.new(@tree, self))
|
556
558
|
end
|
557
559
|
|
558
|
-
@tree.node_cache.insert(self)
|
559
|
-
|
560
560
|
node
|
561
561
|
end
|
562
562
|
|
563
563
|
def next_sibling=(node)
|
564
|
-
@next_sibling = node
|
565
564
|
@tree.node_cache.insert(self)
|
565
|
+
@next_sibling = node
|
566
566
|
if node.nil? && @is_leaf
|
567
567
|
# If this node is a leaf node without a next sibling we need to
|
568
568
|
# register it as the last leaf node.
|
@@ -573,25 +573,25 @@ module PEROBS
|
|
573
573
|
end
|
574
574
|
|
575
575
|
def set_child(index, child)
|
576
|
+
@tree.node_cache.insert(self)
|
576
577
|
if child
|
577
578
|
@children[index] = link(child)
|
578
579
|
@children[index].parent = link(self)
|
579
580
|
else
|
580
581
|
@children[index] = nil
|
581
582
|
end
|
582
|
-
@tree.node_cache.insert(self)
|
583
583
|
|
584
584
|
child
|
585
585
|
end
|
586
586
|
|
587
587
|
def trim(idx)
|
588
|
+
@tree.node_cache.insert(self)
|
588
589
|
@keys.slice!(idx, @keys.length - idx)
|
589
590
|
if @is_leaf
|
590
591
|
@values.slice!(idx, @values.length - idx)
|
591
592
|
else
|
592
593
|
@children.slice!(idx + 1, @children.length - idx - 1)
|
593
594
|
end
|
594
|
-
@tree.node_cache.insert(self)
|
595
595
|
end
|
596
596
|
|
597
597
|
# Search the keys of the node that fits the given key. The result is
|
data/lib/perobs/BigArrayNode.rb
CHANGED
@@ -610,7 +610,7 @@ module PEROBS
|
|
610
610
|
end
|
611
611
|
end
|
612
612
|
|
613
|
-
# @param
|
613
|
+
# @param offset [Integer] offset to search the child index for
|
614
614
|
# @return [Integer] Index of the matching offset or the insert position.
|
615
615
|
def search_child_index(offset)
|
616
616
|
# Handle special case for empty offsets list.
|
data/lib/perobs/Cache.rb
CHANGED
@@ -122,12 +122,25 @@ module PEROBS
|
|
122
122
|
# @param id [Integer] ID of the cached PEROBS::ObjectBase
|
123
123
|
def object_by_id(id)
|
124
124
|
idx = id & @mask
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
#
|
129
|
-
|
130
|
-
|
125
|
+
|
126
|
+
if @transaction_stack.empty?
|
127
|
+
# The index is just a hash. We still need to check if the object IDs are
|
128
|
+
# actually the same before we can return the object.
|
129
|
+
if (obj = @writes[idx]) && obj._id == id
|
130
|
+
# The object was in the write cache.
|
131
|
+
return obj
|
132
|
+
end
|
133
|
+
else
|
134
|
+
# During transactions, the read cache is used to provide fast access
|
135
|
+
# to modified objects. But it does not store all modified objects
|
136
|
+
# since there can be hash collisions. So we also have to check all
|
137
|
+
# transaction objects first.
|
138
|
+
if (obj = @transaction_objects[id])
|
139
|
+
return obj
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
if (obj = @reads[idx]) && obj._id == id
|
131
144
|
# The object was in the read cache.
|
132
145
|
return obj
|
133
146
|
end
|
data/lib/perobs/EquiBlobsFile.rb
CHANGED
data/lib/perobs/FlatFile.rb
CHANGED
@@ -221,6 +221,7 @@ module PEROBS
|
|
221
221
|
flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
|
222
222
|
FlatFileBlobHeader.new(@f, addr, flags, raw_obj_bytesize, id, crc).write
|
223
223
|
@f.write(raw_obj)
|
224
|
+
@f.flush
|
224
225
|
if length != -1 && raw_obj_bytesize < length
|
225
226
|
# The new object was not appended and it did not completely fill the
|
226
227
|
# free space. So we have to write a new header to mark the remaining
|
@@ -247,12 +248,11 @@ module PEROBS
|
|
247
248
|
# If we had an existing object stored for the ID we have to mark
|
248
249
|
# this entry as deleted now.
|
249
250
|
old_header.clear_flags
|
251
|
+
@f.flush
|
250
252
|
# And register the newly freed space with the space list.
|
251
253
|
if @space_list.is_open?
|
252
254
|
@space_list.add_space(old_addr, old_header.length)
|
253
255
|
end
|
254
|
-
else
|
255
|
-
@f.flush
|
256
256
|
end
|
257
257
|
rescue IOError => e
|
258
258
|
PEROBS.log.fatal "Cannot write blob for ID #{id} to FlatFileDB: " +
|
@@ -355,7 +355,7 @@ module PEROBS
|
|
355
355
|
valid_blobs = 0
|
356
356
|
|
357
357
|
# Iterate over all entries.
|
358
|
-
@progressmeter.start('
|
358
|
+
@progressmeter.start('Defragmenting blobs file', @f.size) do |pm|
|
359
359
|
each_blob_header do |header|
|
360
360
|
# If we have stumbled over a corrupted blob we treat it similar to a
|
361
361
|
# deleted blob and reuse the space.
|
@@ -578,7 +578,6 @@ module PEROBS
|
|
578
578
|
# Repair the FlatFile. In contrast to the repair functionality in the
|
579
579
|
# check() method this method is much faster. It simply re-creates the
|
580
580
|
# index and space list from the blob file.
|
581
|
-
# @param repair [Boolean] True if errors should be fixed.
|
582
581
|
# @return [Integer] Number of errors found
|
583
582
|
def repair
|
584
583
|
errors = 0
|
@@ -65,7 +65,7 @@ module PEROBS
|
|
65
65
|
end
|
66
66
|
|
67
67
|
# Insert an ID into the page.
|
68
|
-
# @param
|
68
|
+
# @param id [Integer] The ID to store
|
69
69
|
def insert(id)
|
70
70
|
unless @min_id <= id && id <= @max_id
|
71
71
|
raise ArgumentError, "IDs for this page must be between #{@min_id} " +
|
data/lib/perobs/Store.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Store.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015, 2016, 2017, 2018, 2019
|
5
|
+
# Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
|
6
6
|
# by Chris Schlaeger <chris@taskjuggler.org>
|
7
7
|
#
|
8
8
|
# MIT License
|
@@ -27,6 +27,7 @@
|
|
27
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
28
28
|
|
29
29
|
require 'set'
|
30
|
+
require 'monitor'
|
30
31
|
|
31
32
|
require 'perobs/Log'
|
32
33
|
require 'perobs/Handle'
|
@@ -46,7 +47,7 @@ require 'perobs/ConsoleProgressMeter'
|
|
46
47
|
# PErsistent Ruby OBject Store
|
47
48
|
module PEROBS
|
48
49
|
|
49
|
-
Statistics = Struct.new(:in_memory_objects, :root_objects,
|
50
|
+
Statistics = Struct.new(:in_memory_objects, :root_objects,
|
50
51
|
:marked_objects, :swept_objects,
|
51
52
|
:created_objects, :collected_objects)
|
52
53
|
|
@@ -160,9 +161,6 @@ module PEROBS
|
|
160
161
|
# List of PEROBS objects that are currently available as Ruby objects
|
161
162
|
# hashed by their ID.
|
162
163
|
@in_memory_objects = {}
|
163
|
-
# List of objects that were destroyed already but were still found in
|
164
|
-
# the in_memory_objects list. _collect has not yet been called for them.
|
165
|
-
@zombie_objects = {}
|
166
164
|
|
167
165
|
# This objects keeps some counters of interest.
|
168
166
|
@stats = Statistics.new
|
@@ -173,6 +171,9 @@ module PEROBS
|
|
173
171
|
# objects in memory.
|
174
172
|
@cache = Cache.new(options[:cache_bits] || 16)
|
175
173
|
|
174
|
+
# Lock to serialize access to the Store and all stored data.
|
175
|
+
@lock = Monitor.new
|
176
|
+
|
176
177
|
# The named (global) objects IDs hashed by their name
|
177
178
|
unless options[:no_root_objects]
|
178
179
|
unless (@root_objects = object_by_id(0))
|
@@ -243,8 +244,8 @@ module PEROBS
|
|
243
244
|
end
|
244
245
|
end
|
245
246
|
|
246
|
-
@db = @class_map = @in_memory_objects = @
|
247
|
-
@
|
247
|
+
@db = @class_map = @in_memory_objects = @stats = @cache =
|
248
|
+
@root_objects = nil
|
248
249
|
end
|
249
250
|
|
250
251
|
# You need to call this method to create new PEROBS objects that belong to
|
@@ -259,11 +260,13 @@ module PEROBS
|
|
259
260
|
PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
|
260
261
|
end
|
261
262
|
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
263
|
+
@lock.synchronize do
|
264
|
+
obj = _construct_po(klass, _new_id, *args)
|
265
|
+
# Mark the new object as modified so it gets pushed into the database.
|
266
|
+
@cache.cache_write(obj)
|
267
|
+
# Return a POXReference proxy for the newly created object.
|
268
|
+
obj.myself
|
269
|
+
end
|
267
270
|
end
|
268
271
|
|
269
272
|
# For library internal use only!
|
@@ -280,9 +283,11 @@ module PEROBS
|
|
280
283
|
# method was called. This is an alternative to exit() that additionaly
|
281
284
|
# deletes the entire database.
|
282
285
|
def delete_store
|
283
|
-
@
|
284
|
-
|
285
|
-
@
|
286
|
+
@lock.synchronize do
|
287
|
+
@db.delete_database
|
288
|
+
@db = @class_map = @in_memory_objects = @stats = @cache =
|
289
|
+
@root_objects = nil
|
290
|
+
end
|
286
291
|
end
|
287
292
|
|
288
293
|
# Store the provided object under the given name. Use this to make the
|
@@ -294,25 +299,27 @@ module PEROBS
|
|
294
299
|
# @param obj [PEROBS::Object] The object to store
|
295
300
|
# @return [PEROBS::Object] The stored object.
|
296
301
|
def []=(name, obj)
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
+
@lock.synchronize do
|
303
|
+
# If the passed object is nil, we delete the entry if it exists.
|
304
|
+
if obj.nil?
|
305
|
+
@root_objects.delete(name)
|
306
|
+
return nil
|
307
|
+
end
|
302
308
|
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
+
# We only allow derivatives of PEROBS::Object to be stored in the
|
310
|
+
# store.
|
311
|
+
unless obj.is_a?(ObjectBase)
|
312
|
+
PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
|
313
|
+
"is of class #{obj.class}"
|
314
|
+
end
|
309
315
|
|
310
|
-
|
311
|
-
|
312
|
-
|
316
|
+
unless obj.store == self
|
317
|
+
PEROBS.log.fatal 'The object does not belong to this store.'
|
318
|
+
end
|
313
319
|
|
314
|
-
|
315
|
-
|
320
|
+
# Store the name and mark the name list as modified.
|
321
|
+
@root_objects[name] = obj._id
|
322
|
+
end
|
316
323
|
|
317
324
|
obj
|
318
325
|
end
|
@@ -322,28 +329,34 @@ module PEROBS
|
|
322
329
|
# returned.
|
323
330
|
# @return The requested object or nil if it doesn't exist.
|
324
331
|
def [](name)
|
325
|
-
|
326
|
-
|
332
|
+
@lock.synchronize do
|
333
|
+
# Return nil if there is no object with that name.
|
334
|
+
return nil unless (id = @root_objects[name])
|
327
335
|
|
328
|
-
|
336
|
+
POXReference.new(self, id)
|
337
|
+
end
|
329
338
|
end
|
330
339
|
|
331
340
|
# Return a list with all the names of the root objects.
|
332
341
|
# @return [Array of Symbols]
|
333
342
|
def names
|
334
|
-
@
|
343
|
+
@lock.synchronize do
|
344
|
+
@root_objects.keys
|
345
|
+
end
|
335
346
|
end
|
336
347
|
|
337
348
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
338
349
|
# needed.
|
339
350
|
def sync
|
340
|
-
|
341
|
-
@cache.
|
351
|
+
@lock.synchronize do
|
352
|
+
if @cache.in_transaction?
|
353
|
+
@cache.abort_transaction
|
354
|
+
@cache.flush
|
355
|
+
PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
|
356
|
+
Kernel.caller.join("\n")
|
357
|
+
end
|
342
358
|
@cache.flush
|
343
|
-
PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
|
344
|
-
Kernel.caller.join("\n")
|
345
359
|
end
|
346
|
-
@cache.flush
|
347
360
|
end
|
348
361
|
|
349
362
|
# Return the number of object stored in the store. CAVEAT: This method
|
@@ -353,7 +366,9 @@ module PEROBS
|
|
353
366
|
def size
|
354
367
|
# We don't include the Hash that stores the root objects into the object
|
355
368
|
# count.
|
356
|
-
@
|
369
|
+
@lock.synchronize do
|
370
|
+
@db.item_counter - 1
|
371
|
+
end
|
357
372
|
end
|
358
373
|
|
359
374
|
# Discard all objects that are not somehow connected to the root objects
|
@@ -362,51 +377,20 @@ module PEROBS
|
|
362
377
|
# method periodically.
|
363
378
|
# @return [Integer] The number of collected objects
|
364
379
|
def gc
|
365
|
-
|
366
|
-
|
367
|
-
|
380
|
+
@lock.synchronize do
|
381
|
+
sync
|
382
|
+
mark
|
383
|
+
sweep
|
384
|
+
end
|
368
385
|
end
|
369
386
|
|
370
387
|
# Return the object with the provided ID. This method is not part of the
|
371
388
|
# public API and should never be called by outside users. It's purely
|
372
389
|
# intended for internal use.
|
373
390
|
def object_by_id(id)
|
374
|
-
|
375
|
-
|
376
|
-
begin
|
377
|
-
object = ObjectSpace._id2ref(ruby_object_id)
|
378
|
-
# Let's make sure the object is really the object we are looking
|
379
|
-
# for. The GC might have recycled it already and the Ruby object ID
|
380
|
-
# could now be used for another object.
|
381
|
-
if object.is_a?(ObjectBase) && object._id == id
|
382
|
-
return object
|
383
|
-
end
|
384
|
-
rescue RangeError => e
|
385
|
-
# Due to a race condition the object can still be in the
|
386
|
-
# @in_memory_objects list but has been collected already by the Ruby
|
387
|
-
# GC. In that case we need to load it again. The _collect() call
|
388
|
-
# will happen much later, potentially after we have registered a new
|
389
|
-
# object with the same ID.
|
390
|
-
@zombie_objects[id] = @in_memory_objects.delete(id)
|
391
|
-
end
|
392
|
-
end
|
393
|
-
|
394
|
-
if (obj = @cache.object_by_id(id))
|
395
|
-
PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
|
391
|
+
@lock.synchronize do
|
392
|
+
object_by_id_internal(id)
|
396
393
|
end
|
397
|
-
|
398
|
-
# We don't have the object in memory. Let's find it in the storage.
|
399
|
-
if @db.include?(id)
|
400
|
-
# Great, object found. Read it into memory and return it.
|
401
|
-
obj = ObjectBase::read(self, id)
|
402
|
-
# Add the object to the in-memory storage list.
|
403
|
-
@cache.cache_read(obj)
|
404
|
-
|
405
|
-
return obj
|
406
|
-
end
|
407
|
-
|
408
|
-
# The requested object does not exist. Return nil.
|
409
|
-
nil
|
410
394
|
end
|
411
395
|
|
412
396
|
# This method can be used to check the database and optionally repair it.
|
@@ -471,38 +455,40 @@ module PEROBS
|
|
471
455
|
# beginning of the transaction. The exception is passed on to the
|
472
456
|
# enclosing scope, so you probably want to handle it accordingly.
|
473
457
|
def transaction
|
474
|
-
@cache.begin_transaction
|
458
|
+
@lock.synchronize { @cache.begin_transaction }
|
475
459
|
begin
|
476
460
|
yield if block_given?
|
477
461
|
rescue => e
|
478
|
-
@cache.abort_transaction
|
462
|
+
@lock.synchronize { @cache.abort_transaction }
|
479
463
|
raise e
|
480
464
|
end
|
481
|
-
@cache.end_transaction
|
465
|
+
@lock.synchronize { @cache.end_transaction }
|
482
466
|
end
|
483
467
|
|
484
468
|
# Calls the given block once for each object, passing that object as a
|
485
469
|
# parameter.
|
486
470
|
def each
|
487
|
-
@
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
"
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
471
|
+
@lock.synchronize do
|
472
|
+
@db.clear_marks
|
473
|
+
# Start with the object 0 and the indexes of the root objects. Push them
|
474
|
+
# onto the work stack.
|
475
|
+
stack = [ 0 ] + @root_objects.values
|
476
|
+
while !stack.empty?
|
477
|
+
# Get an object index from the stack.
|
478
|
+
id = stack.pop
|
479
|
+
next if @db.is_marked?(id)
|
480
|
+
|
481
|
+
unless (obj = object_by_id_internal(id))
|
482
|
+
PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
|
483
|
+
"not found."
|
484
|
+
end
|
485
|
+
# Mark the object so it will never be pushed to the stack again.
|
486
|
+
@db.mark(id)
|
487
|
+
yield(obj.myself) if block_given?
|
488
|
+
# Push the IDs of all unmarked referenced objects onto the stack
|
489
|
+
obj._referenced_object_ids.each do |r_id|
|
490
|
+
stack << r_id unless @db.is_marked?(r_id)
|
491
|
+
end
|
506
492
|
end
|
507
493
|
end
|
508
494
|
end
|
@@ -510,7 +496,7 @@ module PEROBS
|
|
510
496
|
# Rename classes of objects stored in the data base.
|
511
497
|
# @param rename_map [Hash] Hash that maps the old name to the new name
|
512
498
|
def rename_classes(rename_map)
|
513
|
-
@class_map.rename(rename_map)
|
499
|
+
@lock.synchronize { @class_map.rename(rename_map) }
|
514
500
|
end
|
515
501
|
|
516
502
|
# Internal method. Don't use this outside of this library!
|
@@ -518,14 +504,16 @@ module PEROBS
|
|
518
504
|
# random numbers between 0 and 2**64 - 1.
|
519
505
|
# @return [Integer]
|
520
506
|
def _new_id
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
507
|
+
@lock.synchronize do
|
508
|
+
begin
|
509
|
+
# Generate a random number. It's recommended to not store more than
|
510
|
+
# 2**62 objects in the same store.
|
511
|
+
id = rand(2**64)
|
512
|
+
# Ensure that we don't have already another object with this ID.
|
513
|
+
end while @in_memory_objects.include?(id) || @db.include?(id)
|
527
514
|
|
528
|
-
|
515
|
+
id
|
516
|
+
end
|
529
517
|
end
|
530
518
|
|
531
519
|
# Internal method. Don't use this outside of this library!
|
@@ -536,16 +524,18 @@ module PEROBS
|
|
536
524
|
# @param obj [BasicObject] Object to register
|
537
525
|
# @param id [Integer] object ID
|
538
526
|
def _register_in_memory(obj, id)
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
"
|
545
|
-
|
527
|
+
@lock.synchronize do
|
528
|
+
unless obj.is_a?(ObjectBase)
|
529
|
+
PEROBS.log.fatal "You can only register ObjectBase objects"
|
530
|
+
end
|
531
|
+
if @in_memory_objects.include?(id)
|
532
|
+
PEROBS.log.fatal "The Store::_in_memory_objects list already " +
|
533
|
+
"contains an object for ID #{id}"
|
534
|
+
end
|
546
535
|
|
547
|
-
|
548
|
-
|
536
|
+
@in_memory_objects[id] = obj.object_id
|
537
|
+
@stats[:created_objects] += 1
|
538
|
+
end
|
549
539
|
end
|
550
540
|
|
551
541
|
# Remove the object from the in-memory list. This is an internal method
|
@@ -553,26 +543,73 @@ module PEROBS
|
|
553
543
|
# finalizer, so many restrictions apply!
|
554
544
|
# @param id [Integer] Object ID of object to remove from the list
|
555
545
|
def _collect(id, ruby_object_id)
|
546
|
+
# This method should only be called from the Ruby garbage collector.
|
547
|
+
# Therefor no locking is needed or even possible. The GC can kick in at
|
548
|
+
# any time and we could be anywhere in the code. So there is a small
|
549
|
+
# risk for a race here, but it should not have any serious consequences.
|
556
550
|
if @in_memory_objects[id] == ruby_object_id
|
557
551
|
@in_memory_objects.delete(id)
|
558
552
|
@stats[:collected_objects] += 1
|
559
|
-
elsif @zombie_objects[id] == ruby_object_id
|
560
|
-
@zombie_objects.delete(id)
|
561
|
-
@stats[:collected_objects] += 1
|
562
553
|
end
|
563
554
|
end
|
564
555
|
|
565
556
|
# This method returns a Hash with some statistics about this store.
|
566
557
|
def statistics
|
567
|
-
@
|
568
|
-
|
569
|
-
|
558
|
+
@lock.synchronize do
|
559
|
+
@stats.in_memory_objects = @in_memory_objects.length
|
560
|
+
@stats.root_objects = @root_objects.length
|
561
|
+
end
|
570
562
|
|
571
563
|
@stats
|
572
564
|
end
|
573
565
|
|
574
566
|
private
|
575
567
|
|
568
|
+
def object_by_id_internal(id)
|
569
|
+
if (ruby_object_id = @in_memory_objects[id])
|
570
|
+
# We have the object in memory so we can just return it.
|
571
|
+
begin
|
572
|
+
object = ObjectSpace._id2ref(ruby_object_id)
|
573
|
+
# Let's make sure the object is really the object we are looking
|
574
|
+
# for. The GC might have recycled it already and the Ruby object ID
|
575
|
+
# could now be used for another object.
|
576
|
+
if object.is_a?(ObjectBase) && object._id == id
|
577
|
+
return object
|
578
|
+
end
|
579
|
+
rescue RangeError => e
|
580
|
+
# Due to a race condition the object can still be in the
|
581
|
+
# @in_memory_objects list but has been collected already by the Ruby
|
582
|
+
# GC. The _collect() call has not been completed yet. We now have to
|
583
|
+
# wait until this has been done. I think the GC lock will prevent a
|
584
|
+
# race on @in_memory_objects.
|
585
|
+
GC.start
|
586
|
+
while @in_memory_objects.include?(id)
|
587
|
+
sleep 0.01
|
588
|
+
end
|
589
|
+
end
|
590
|
+
end
|
591
|
+
|
592
|
+
# This is just a safety check. It has never triggered, so we can disable
|
593
|
+
# it for now.
|
594
|
+
#if (obj = @cache.object_by_id(id))
|
595
|
+
# PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
|
596
|
+
# "cache but not in_memory"
|
597
|
+
#end
|
598
|
+
|
599
|
+
# We don't have the object in memory. Let's find it in the storage.
|
600
|
+
if @db.include?(id)
|
601
|
+
# Great, object found. Read it into memory and return it.
|
602
|
+
obj = ObjectBase::read(self, id)
|
603
|
+
# Add the object to the in-memory storage list.
|
604
|
+
@cache.cache_read(obj)
|
605
|
+
|
606
|
+
return obj
|
607
|
+
end
|
608
|
+
|
609
|
+
# The requested object does not exist. Return nil.
|
610
|
+
nil
|
611
|
+
end
|
612
|
+
|
576
613
|
# Mark phase of a mark-and-sweep garbage collector. It will mark all
|
577
614
|
# objects that are reachable from the root objects.
|
578
615
|
def mark
|
data/lib/perobs/version.rb
CHANGED
data/perobs.gemspec
CHANGED
@@ -20,5 +20,5 @@ GEM_SPEC = Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_development_dependency 'bundler', '~> 2.3'
|
22
22
|
spec.add_development_dependency 'yard', '~>0.9.12'
|
23
|
-
spec.add_development_dependency 'rake', '~>
|
23
|
+
spec.add_development_dependency 'rake', '~> 13.0.3'
|
24
24
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 13.0.3
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 13.0.3
|
55
55
|
description: Library to provide a persistent object store
|
56
56
|
email:
|
57
57
|
- chris@linux.com
|
@@ -70,7 +70,6 @@ files:
|
|
70
70
|
- lib/perobs/BTreeBlob.rb
|
71
71
|
- lib/perobs/BTreeDB.rb
|
72
72
|
- lib/perobs/BTreeNode.rb
|
73
|
-
- lib/perobs/BTreeNodeCache.rb
|
74
73
|
- lib/perobs/BTreeNodeLink.rb
|
75
74
|
- lib/perobs/BigArray.rb
|
76
75
|
- lib/perobs/BigArrayNode.rb
|
@@ -162,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
162
161
|
- !ruby/object:Gem::Version
|
163
162
|
version: '0'
|
164
163
|
requirements: []
|
165
|
-
rubygems_version: 3.2.
|
164
|
+
rubygems_version: 3.2.32
|
166
165
|
signing_key:
|
167
166
|
specification_version: 4
|
168
167
|
summary: Persistent Ruby Object Store
|
@@ -1,109 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
|
-
# = BTree.rb -- Persistent Ruby Object Store
|
4
|
-
#
|
5
|
-
# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
-
#
|
7
|
-
# MIT License
|
8
|
-
#
|
9
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
-
# a copy of this software and associated documentation files (the
|
11
|
-
# "Software"), to deal in the Software without restriction, including
|
12
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
-
# the following conditions:
|
16
|
-
#
|
17
|
-
# The above copyright notice and this permission notice shall be
|
18
|
-
# included in all copies or substantial portions of the Software.
|
19
|
-
#
|
20
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
-
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
-
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
-
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
-
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
-
|
28
|
-
require 'perobs/BTreeNode'
|
29
|
-
|
30
|
-
module PEROBS
|
31
|
-
|
32
|
-
class BTreeNodeCache
|
33
|
-
|
34
|
-
def initialize(tree)
|
35
|
-
@tree = tree
|
36
|
-
clear
|
37
|
-
end
|
38
|
-
|
39
|
-
def get(address)
|
40
|
-
if (node = @modified_nodes[address])
|
41
|
-
return node
|
42
|
-
end
|
43
|
-
|
44
|
-
if (node = @top_nodes[address])
|
45
|
-
return node
|
46
|
-
end
|
47
|
-
|
48
|
-
if (node = @ephemeral_nodes[address])
|
49
|
-
return node
|
50
|
-
end
|
51
|
-
|
52
|
-
BTreeNode::load(@tree, address)
|
53
|
-
end
|
54
|
-
|
55
|
-
def set_root(node)
|
56
|
-
node = node.get_node if node.is_a?(BTreeNodeLink)
|
57
|
-
|
58
|
-
@top_nodes = {}
|
59
|
-
@top_nodes[node.node_address] = node
|
60
|
-
end
|
61
|
-
|
62
|
-
def insert(node, modified = true)
|
63
|
-
unless node
|
64
|
-
PEROBS.log.fatal "nil cannot be cached"
|
65
|
-
end
|
66
|
-
node = node.get_node if node.is_a?(BTreeNodeLink)
|
67
|
-
|
68
|
-
if modified
|
69
|
-
@modified_nodes[node.node_address] = node
|
70
|
-
end
|
71
|
-
@ephemeral_nodes[node.node_address] = node
|
72
|
-
|
73
|
-
if !@top_nodes.include?(node) && node.is_top?
|
74
|
-
@top_nodes[node.node_address] = node
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
def _collect(address, ruby_object_id)
|
79
|
-
# Just a dummy for now
|
80
|
-
end
|
81
|
-
|
82
|
-
# Remove a node from the cache.
|
83
|
-
# @param address [Integer] address of node to remove.
|
84
|
-
def delete(address)
|
85
|
-
@ephemeral_nodes.delete(address)
|
86
|
-
@top_nodes.delete(address)
|
87
|
-
@modified_nodes.delete(address)
|
88
|
-
end
|
89
|
-
|
90
|
-
# Flush all dirty nodes into the backing store.
|
91
|
-
def flush(now = false)
|
92
|
-
if now || @modified_nodes.size > 1024
|
93
|
-
@modified_nodes.each_value { |node| node.write_node }
|
94
|
-
@modified_nodes = {}
|
95
|
-
end
|
96
|
-
@ephemeral_nodes = {}
|
97
|
-
end
|
98
|
-
|
99
|
-
# Remove all nodes from the cache.
|
100
|
-
def clear
|
101
|
-
@top_nodes = {}
|
102
|
-
@ephemeral_nodes = {}
|
103
|
-
@modified_nodes = {}
|
104
|
-
end
|
105
|
-
|
106
|
-
end
|
107
|
-
|
108
|
-
end
|
109
|
-
|