perobs 4.0.0 → 4.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +27 -16
- data/lib/perobs/Array.rb +66 -19
- data/lib/perobs/BTree.rb +106 -15
- data/lib/perobs/BTreeBlob.rb +4 -3
- data/lib/perobs/BTreeDB.rb +5 -4
- data/lib/perobs/BTreeNode.rb +482 -156
- data/lib/perobs/BTreeNodeLink.rb +10 -0
- data/lib/perobs/BigArray.rb +285 -0
- data/lib/perobs/BigArrayNode.rb +1002 -0
- data/lib/perobs/BigHash.rb +246 -0
- data/lib/perobs/BigTree.rb +197 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/Cache.rb +48 -10
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +57 -15
- data/lib/perobs/EquiBlobsFile.rb +155 -50
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +519 -227
- data/lib/perobs/FlatFileBlobHeader.rb +113 -54
- data/lib/perobs/FlatFileDB.rb +49 -23
- data/lib/perobs/FuzzyStringMatcher.rb +175 -0
- data/lib/perobs/Hash.rb +127 -33
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/Object.rb +18 -15
- data/lib/perobs/ObjectBase.rb +46 -5
- data/lib/perobs/PersistentObjectCache.rb +57 -68
- data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceManager.rb +273 -0
- data/lib/perobs/SpaceTree.rb +21 -12
- data/lib/perobs/SpaceTreeNode.rb +53 -61
- data/lib/perobs/Store.rb +264 -145
- data/lib/perobs/version.rb +1 -1
- data/lib/perobs.rb +2 -0
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +15 -6
- data/test/BTree_spec.rb +6 -2
- data/test/BigArray_spec.rb +261 -0
- data/test/BigHash_spec.rb +152 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -1
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +198 -14
- data/test/FuzzyStringMatcher_spec.rb +261 -0
- data/test/Hash_spec.rb +13 -3
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +155 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/SpaceManager_spec.rb +176 -0
- data/test/SpaceTree_spec.rb +4 -1
- data/test/Store_spec.rb +305 -203
- data/test/spec_helper.rb +9 -4
- metadata +57 -16
- data/lib/perobs/BTreeNodeCache.rb +0 -109
- data/lib/perobs/TreeDB.rb +0 -277
data/lib/perobs/Store.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
#
|
3
3
|
# = Store.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015, 2016
|
5
|
+
# Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
|
6
|
+
# by Chris Schlaeger <chris@taskjuggler.org>
|
6
7
|
#
|
7
8
|
# MIT License
|
8
9
|
#
|
@@ -26,6 +27,7 @@
|
|
26
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
28
|
|
28
29
|
require 'set'
|
30
|
+
require 'monitor'
|
29
31
|
|
30
32
|
require 'perobs/Log'
|
31
33
|
require 'perobs/Handle'
|
@@ -36,12 +38,18 @@ require 'perobs/FlatFileDB'
|
|
36
38
|
require 'perobs/Object'
|
37
39
|
require 'perobs/Hash'
|
38
40
|
require 'perobs/Array'
|
41
|
+
require 'perobs/BigTree'
|
42
|
+
require 'perobs/BigHash'
|
43
|
+
require 'perobs/BigArray'
|
44
|
+
require 'perobs/ProgressMeter'
|
45
|
+
require 'perobs/ConsoleProgressMeter'
|
39
46
|
|
40
47
|
# PErsistent Ruby OBject Store
|
41
48
|
module PEROBS
|
42
49
|
|
43
50
|
Statistics = Struct.new(:in_memory_objects, :root_objects,
|
44
|
-
:marked_objects, :swept_objects
|
51
|
+
:marked_objects, :swept_objects,
|
52
|
+
:created_objects, :collected_objects)
|
45
53
|
|
46
54
|
# PEROBS::Store is a persistent storage system for Ruby objects. Regular
|
47
55
|
# Ruby objects are transparently stored in a back-end storage and retrieved
|
@@ -103,6 +111,7 @@ module PEROBS
|
|
103
111
|
class Store
|
104
112
|
|
105
113
|
attr_reader :db, :cache, :class_map
|
114
|
+
attr_writer :root_objects
|
106
115
|
|
107
116
|
# Create a new Store.
|
108
117
|
# @param data_base [String] the name of the database
|
@@ -132,8 +141,17 @@ module PEROBS
|
|
132
141
|
# :yaml : Can also handle most Ruby data types and is
|
133
142
|
# portable between Ruby versions (1.9 and later).
|
134
143
|
# Unfortunately, it is 10x slower than marshal.
|
144
|
+
# :progressmeter : reference to a ProgressMeter object that receives
|
145
|
+
# progress information during longer running tasks.
|
146
|
+
# It defaults to ProgressMeter which only logs into
|
147
|
+
# the log. Use ConsoleProgressMeter or a derived
|
148
|
+
# class for more fancy progress reporting.
|
149
|
+
# :no_root_objects : Create a new store without root objects. This only
|
150
|
+
# makes sense if you want to copy the objects of
|
151
|
+
# another store into this store.
|
135
152
|
def initialize(data_base, options = {})
|
136
153
|
# Create a backing store handler
|
154
|
+
@progressmeter = (options[:progressmeter] ||= ProgressMeter.new)
|
137
155
|
@db = (options[:engine] || FlatFileDB).new(data_base, options)
|
138
156
|
@db.open
|
139
157
|
# Create a map that can translate classes to numerical IDs and vice
|
@@ -146,22 +164,29 @@ module PEROBS
|
|
146
164
|
|
147
165
|
# This objects keeps some counters of interest.
|
148
166
|
@stats = Statistics.new
|
167
|
+
@stats[:created_objects] = 0
|
168
|
+
@stats[:collected_objects] = 0
|
149
169
|
|
150
170
|
# The Cache reduces read and write latencies by keeping a subset of the
|
151
171
|
# objects in memory.
|
152
172
|
@cache = Cache.new(options[:cache_bits] || 16)
|
153
173
|
|
174
|
+
# Lock to serialize access to the Store and all stored data.
|
175
|
+
@lock = Monitor.new
|
176
|
+
|
154
177
|
# The named (global) objects IDs hashed by their name
|
155
|
-
unless
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
"
|
178
|
+
unless options[:no_root_objects]
|
179
|
+
unless (@root_objects = object_by_id(0))
|
180
|
+
PEROBS.log.debug "Initializing the PEROBS store"
|
181
|
+
# The root object hash always has the object ID 0.
|
182
|
+
@root_objects = _construct_po(Hash, 0)
|
183
|
+
# Mark the root_objects object as modified.
|
184
|
+
@cache.cache_write(@root_objects)
|
185
|
+
end
|
186
|
+
unless @root_objects.is_a?(Hash)
|
187
|
+
PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
|
188
|
+
"but is a #{@root_objects.class}"
|
189
|
+
end
|
165
190
|
end
|
166
191
|
end
|
167
192
|
|
@@ -173,7 +198,9 @@ module PEROBS
|
|
173
198
|
sync
|
174
199
|
|
175
200
|
# Create a new store with the specified directory and options.
|
176
|
-
|
201
|
+
new_options = options.clone
|
202
|
+
new_options[:no_root_objects] = true
|
203
|
+
new_db = Store.new(dir, new_options)
|
177
204
|
# Clear the cache.
|
178
205
|
new_db.sync
|
179
206
|
# Copy all objects of the existing store to the new store.
|
@@ -184,6 +211,7 @@ module PEROBS
|
|
184
211
|
obj._sync
|
185
212
|
i += 1
|
186
213
|
end
|
214
|
+
new_db.root_objects = new_db.object_by_id(0)
|
187
215
|
PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
|
188
216
|
# Flush the new store and close it.
|
189
217
|
new_db.exit
|
@@ -191,20 +219,34 @@ module PEROBS
|
|
191
219
|
true
|
192
220
|
end
|
193
221
|
|
194
|
-
|
195
222
|
# Close the store and ensure that all in-memory objects are written out to
|
196
223
|
# the storage backend. The Store object is no longer usable after this
|
197
224
|
# method was called.
|
198
225
|
def exit
|
199
226
|
if @cache && @cache.in_transaction?
|
200
|
-
|
227
|
+
@cache.abort_transaction
|
228
|
+
@cache.flush
|
229
|
+
@db.close if @db
|
230
|
+
PEROBS.log.fatal "You cannot call exit() during a transaction: #{Kernel.caller}"
|
201
231
|
end
|
202
232
|
@cache.flush if @cache
|
203
233
|
@db.close if @db
|
204
|
-
@db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
|
205
|
-
nil
|
206
|
-
end
|
207
234
|
|
235
|
+
GC.start
|
236
|
+
if @stats
|
237
|
+
unless @stats[:created_objects] == @stats[:collected_objects] +
|
238
|
+
@in_memory_objects.length
|
239
|
+
PEROGS.log.fatal "Created objects count " +
|
240
|
+
"(#{@stats[:created_objects]})" +
|
241
|
+
" is not equal to the collected count " +
|
242
|
+
"(#{@stats[:collected_objects]}) + in_memory_objects count " +
|
243
|
+
"(#{@in_memory_objects.length})"
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
@db = @class_map = @in_memory_objects = @stats = @cache =
|
248
|
+
@root_objects = nil
|
249
|
+
end
|
208
250
|
|
209
251
|
# You need to call this method to create new PEROBS objects that belong to
|
210
252
|
# this Store.
|
@@ -218,11 +260,13 @@ module PEROBS
|
|
218
260
|
PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
|
219
261
|
end
|
220
262
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
263
|
+
@lock.synchronize do
|
264
|
+
obj = _construct_po(klass, _new_id, *args)
|
265
|
+
# Mark the new object as modified so it gets pushed into the database.
|
266
|
+
@cache.cache_write(obj)
|
267
|
+
# Return a POXReference proxy for the newly created object.
|
268
|
+
obj.myself
|
269
|
+
end
|
226
270
|
end
|
227
271
|
|
228
272
|
# For library internal use only!
|
@@ -239,9 +283,11 @@ module PEROBS
|
|
239
283
|
# method was called. This is an alternative to exit() that additionaly
|
240
284
|
# deletes the entire database.
|
241
285
|
def delete_store
|
242
|
-
@
|
243
|
-
|
244
|
-
|
286
|
+
@lock.synchronize do
|
287
|
+
@db.delete_database
|
288
|
+
@db = @class_map = @in_memory_objects = @stats = @cache =
|
289
|
+
@root_objects = nil
|
290
|
+
end
|
245
291
|
end
|
246
292
|
|
247
293
|
# Store the provided object under the given name. Use this to make the
|
@@ -253,25 +299,27 @@ module PEROBS
|
|
253
299
|
# @param obj [PEROBS::Object] The object to store
|
254
300
|
# @return [PEROBS::Object] The stored object.
|
255
301
|
def []=(name, obj)
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
302
|
+
@lock.synchronize do
|
303
|
+
# If the passed object is nil, we delete the entry if it exists.
|
304
|
+
if obj.nil?
|
305
|
+
@root_objects.delete(name)
|
306
|
+
return nil
|
307
|
+
end
|
261
308
|
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
309
|
+
# We only allow derivatives of PEROBS::Object to be stored in the
|
310
|
+
# store.
|
311
|
+
unless obj.is_a?(ObjectBase)
|
312
|
+
PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
|
313
|
+
"is of class #{obj.class}"
|
314
|
+
end
|
268
315
|
|
269
|
-
|
270
|
-
|
271
|
-
|
316
|
+
unless obj.store == self
|
317
|
+
PEROBS.log.fatal 'The object does not belong to this store.'
|
318
|
+
end
|
272
319
|
|
273
|
-
|
274
|
-
|
320
|
+
# Store the name and mark the name list as modified.
|
321
|
+
@root_objects[name] = obj._id
|
322
|
+
end
|
275
323
|
|
276
324
|
obj
|
277
325
|
end
|
@@ -281,25 +329,46 @@ module PEROBS
|
|
281
329
|
# returned.
|
282
330
|
# @return The requested object or nil if it doesn't exist.
|
283
331
|
def [](name)
|
284
|
-
|
285
|
-
|
332
|
+
@lock.synchronize do
|
333
|
+
# Return nil if there is no object with that name.
|
334
|
+
return nil unless (id = @root_objects[name])
|
286
335
|
|
287
|
-
|
336
|
+
POXReference.new(self, id)
|
337
|
+
end
|
288
338
|
end
|
289
339
|
|
290
340
|
# Return a list with all the names of the root objects.
|
291
341
|
# @return [Array of Symbols]
|
292
342
|
def names
|
293
|
-
@
|
343
|
+
@lock.synchronize do
|
344
|
+
@root_objects.keys
|
345
|
+
end
|
294
346
|
end
|
295
347
|
|
296
348
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
297
349
|
# needed.
|
298
350
|
def sync
|
299
|
-
|
300
|
-
|
351
|
+
@lock.synchronize do
|
352
|
+
if @cache.in_transaction?
|
353
|
+
@cache.abort_transaction
|
354
|
+
@cache.flush
|
355
|
+
PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
|
356
|
+
Kernel.caller.join("\n")
|
357
|
+
end
|
358
|
+
@cache.flush
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
# Return the number of object stored in the store. CAVEAT: This method
|
363
|
+
# will only return correct values when it is separated from any mutating
|
364
|
+
# call by a call to sync().
|
365
|
+
# @return [Integer] Number of persistently stored objects in the Store.
|
366
|
+
def size
|
367
|
+
# We don't include the Hash that stores the root objects into the object
|
368
|
+
# count.
|
369
|
+
@lock.synchronize do
|
370
|
+
@db.item_counter - 1
|
301
371
|
end
|
302
|
-
@cache.flush
|
303
372
|
end
|
304
373
|
|
305
374
|
# Discard all objects that are not somehow connected to the root objects
|
@@ -308,58 +377,20 @@ module PEROBS
|
|
308
377
|
# method periodically.
|
309
378
|
# @return [Integer] The number of collected objects
|
310
379
|
def gc
|
311
|
-
|
312
|
-
|
380
|
+
@lock.synchronize do
|
381
|
+
sync
|
382
|
+
mark
|
383
|
+
sweep
|
313
384
|
end
|
314
|
-
sync
|
315
|
-
mark
|
316
|
-
sweep
|
317
385
|
end
|
318
386
|
|
319
387
|
# Return the object with the provided ID. This method is not part of the
|
320
388
|
# public API and should never be called by outside users. It's purely
|
321
389
|
# intended for internal use.
|
322
390
|
def object_by_id(id)
|
323
|
-
|
324
|
-
|
325
|
-
begin
|
326
|
-
object = ObjectSpace._id2ref(ruby_object_id)
|
327
|
-
# Let's make sure the object is really the object we are looking
|
328
|
-
# for. The GC might have recycled it already and the Ruby object ID
|
329
|
-
# could now be used for another object.
|
330
|
-
if object.is_a?(ObjectBase) && object._id == id
|
331
|
-
return object
|
332
|
-
end
|
333
|
-
rescue RangeError => e
|
334
|
-
# Due to a race condition the object can still be in the
|
335
|
-
# @in_memory_objects list but has been collected already by the Ruby
|
336
|
-
# GC. In that case we need to load it again. In this case the
|
337
|
-
# _collect() call will happen much later, potentially after we have
|
338
|
-
# registered a new object with the same ID.
|
339
|
-
@in_memory_objects.delete(id)
|
340
|
-
end
|
391
|
+
@lock.synchronize do
|
392
|
+
object_by_id_internal(id)
|
341
393
|
end
|
342
|
-
|
343
|
-
if (obj = @cache.object_by_id(id))
|
344
|
-
PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
|
345
|
-
end
|
346
|
-
|
347
|
-
# We don't have the object in memory. Let's find it in the storage.
|
348
|
-
if @db.include?(id)
|
349
|
-
# Great, object found. Read it into memory and return it.
|
350
|
-
obj = ObjectBase::read(self, id)
|
351
|
-
# Add the object to the in-memory storage list.
|
352
|
-
@cache.cache_read(obj)
|
353
|
-
|
354
|
-
return obj
|
355
|
-
end
|
356
|
-
|
357
|
-
#if (obj = @db.search_object(id))
|
358
|
-
# PEROBS.log.fatal "Object was not in index but in DB"
|
359
|
-
#end
|
360
|
-
|
361
|
-
# The requested object does not exist. Return nil.
|
362
|
-
nil
|
363
394
|
end
|
364
395
|
|
365
396
|
# This method can be used to check the database and optionally repair it.
|
@@ -370,38 +401,42 @@ module PEROBS
|
|
370
401
|
# made.
|
371
402
|
# @return [Integer] The number of references to bad objects found.
|
372
403
|
def check(repair = false)
|
404
|
+
stats = { :errors => 0, :object_cnt => 0 }
|
405
|
+
|
373
406
|
# All objects must have in-db version.
|
374
407
|
sync
|
375
408
|
# Run basic consistency checks first.
|
376
|
-
errors
|
409
|
+
stats[:errors] += @db.check_db(repair)
|
377
410
|
|
378
411
|
# We will use the mark to mark all objects that we have checked already.
|
379
412
|
# Before we start, we need to clear all marks.
|
380
413
|
@db.clear_marks
|
381
414
|
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
415
|
+
@progressmeter.start("Checking object link structure",
|
416
|
+
@db.item_counter) do
|
417
|
+
@root_objects.each do |name, id|
|
418
|
+
check_object(id, repair, stats)
|
419
|
+
end
|
386
420
|
end
|
387
421
|
|
388
422
|
# Delete all broken root objects.
|
389
423
|
if repair
|
390
424
|
@root_objects.delete_if do |name, id|
|
391
|
-
unless
|
425
|
+
unless @db.check(id, repair)
|
392
426
|
PEROBS.log.error "Discarding broken root object '#{name}' " +
|
393
427
|
"with ID #{id}"
|
394
|
-
errors += 1
|
428
|
+
stats[:errors] += 1
|
395
429
|
end
|
396
|
-
!res
|
397
430
|
end
|
398
431
|
end
|
399
432
|
|
400
|
-
if errors > 0
|
433
|
+
if stats[:errors] > 0
|
401
434
|
if repair
|
402
|
-
PEROBS.log.error "#{errors} errors found in
|
435
|
+
PEROBS.log.error "#{stats[:errors]} errors found in " +
|
436
|
+
"#{stats[:object_cnt]} objects"
|
403
437
|
else
|
404
|
-
PEROBS.log.fatal "#{errors} errors found in
|
438
|
+
PEROBS.log.fatal "#{stats[:errors]} errors found in " +
|
439
|
+
"#{stats[:object_cnt]} objects"
|
405
440
|
end
|
406
441
|
else
|
407
442
|
PEROBS.log.debug "No errors found"
|
@@ -410,7 +445,7 @@ module PEROBS
|
|
410
445
|
# Ensure that any fixes are written into the DB.
|
411
446
|
sync if repair
|
412
447
|
|
413
|
-
errors
|
448
|
+
stats[:errors]
|
414
449
|
end
|
415
450
|
|
416
451
|
# This method will execute the provided block as an atomic transaction
|
@@ -420,35 +455,40 @@ module PEROBS
|
|
420
455
|
# beginning of the transaction. The exception is passed on to the
|
421
456
|
# enclosing scope, so you probably want to handle it accordingly.
|
422
457
|
def transaction
|
423
|
-
@cache.begin_transaction
|
458
|
+
@lock.synchronize { @cache.begin_transaction }
|
424
459
|
begin
|
425
460
|
yield if block_given?
|
426
461
|
rescue => e
|
427
|
-
@cache.abort_transaction
|
462
|
+
@lock.synchronize { @cache.abort_transaction }
|
428
463
|
raise e
|
429
464
|
end
|
430
|
-
@cache.end_transaction
|
465
|
+
@lock.synchronize { @cache.end_transaction }
|
431
466
|
end
|
432
467
|
|
433
468
|
# Calls the given block once for each object, passing that object as a
|
434
469
|
# parameter.
|
435
470
|
def each
|
436
|
-
@
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
471
|
+
@lock.synchronize do
|
472
|
+
@db.clear_marks
|
473
|
+
# Start with the object 0 and the indexes of the root objects. Push them
|
474
|
+
# onto the work stack.
|
475
|
+
stack = [ 0 ] + @root_objects.values
|
476
|
+
while !stack.empty?
|
477
|
+
# Get an object index from the stack.
|
478
|
+
id = stack.pop
|
479
|
+
next if @db.is_marked?(id)
|
480
|
+
|
481
|
+
unless (obj = object_by_id_internal(id))
|
482
|
+
PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
|
483
|
+
"not found."
|
484
|
+
end
|
485
|
+
# Mark the object so it will never be pushed to the stack again.
|
486
|
+
@db.mark(id)
|
487
|
+
yield(obj.myself) if block_given?
|
488
|
+
# Push the IDs of all unmarked referenced objects onto the stack
|
489
|
+
obj._referenced_object_ids.each do |r_id|
|
490
|
+
stack << r_id unless @db.is_marked?(r_id)
|
491
|
+
end
|
452
492
|
end
|
453
493
|
end
|
454
494
|
end
|
@@ -456,7 +496,7 @@ module PEROBS
|
|
456
496
|
# Rename classes of objects stored in the data base.
|
457
497
|
# @param rename_map [Hash] Hash that maps the old name to the new name
|
458
498
|
def rename_classes(rename_map)
|
459
|
-
@class_map.rename(rename_map)
|
499
|
+
@lock.synchronize { @class_map.rename(rename_map) }
|
460
500
|
end
|
461
501
|
|
462
502
|
# Internal method. Don't use this outside of this library!
|
@@ -464,14 +504,16 @@ module PEROBS
|
|
464
504
|
# random numbers between 0 and 2**64 - 1.
|
465
505
|
# @return [Integer]
|
466
506
|
def _new_id
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
507
|
+
@lock.synchronize do
|
508
|
+
begin
|
509
|
+
# Generate a random number. It's recommended to not store more than
|
510
|
+
# 2**62 objects in the same store.
|
511
|
+
id = rand(2**64)
|
512
|
+
# Ensure that we don't have already another object with this ID.
|
513
|
+
end while @in_memory_objects.include?(id) || @db.include?(id)
|
473
514
|
|
474
|
-
|
515
|
+
id
|
516
|
+
end
|
475
517
|
end
|
476
518
|
|
477
519
|
# Internal method. Don't use this outside of this library!
|
@@ -482,7 +524,18 @@ module PEROBS
|
|
482
524
|
# @param obj [BasicObject] Object to register
|
483
525
|
# @param id [Integer] object ID
|
484
526
|
def _register_in_memory(obj, id)
|
485
|
-
@
|
527
|
+
@lock.synchronize do
|
528
|
+
unless obj.is_a?(ObjectBase)
|
529
|
+
PEROBS.log.fatal "You can only register ObjectBase objects"
|
530
|
+
end
|
531
|
+
if @in_memory_objects.include?(id)
|
532
|
+
PEROBS.log.fatal "The Store::_in_memory_objects list already " +
|
533
|
+
"contains an object for ID #{id}"
|
534
|
+
end
|
535
|
+
|
536
|
+
@in_memory_objects[id] = obj.object_id
|
537
|
+
@stats[:created_objects] += 1
|
538
|
+
end
|
486
539
|
end
|
487
540
|
|
488
541
|
# Remove the object from the in-memory list. This is an internal method
|
@@ -490,40 +543,101 @@ module PEROBS
|
|
490
543
|
# finalizer, so many restrictions apply!
|
491
544
|
# @param id [Integer] Object ID of object to remove from the list
|
492
545
|
def _collect(id, ruby_object_id)
|
546
|
+
# This method should only be called from the Ruby garbage collector.
|
547
|
+
# Therefor no locking is needed or even possible. The GC can kick in at
|
548
|
+
# any time and we could be anywhere in the code. So there is a small
|
549
|
+
# risk for a race here, but it should not have any serious consequences.
|
493
550
|
if @in_memory_objects[id] == ruby_object_id
|
494
551
|
@in_memory_objects.delete(id)
|
552
|
+
@stats[:collected_objects] += 1
|
495
553
|
end
|
496
554
|
end
|
497
555
|
|
498
556
|
# This method returns a Hash with some statistics about this store.
|
499
557
|
def statistics
|
500
|
-
@
|
501
|
-
|
558
|
+
@lock.synchronize do
|
559
|
+
@stats.in_memory_objects = @in_memory_objects.length
|
560
|
+
@stats.root_objects = @root_objects.length
|
561
|
+
end
|
502
562
|
|
503
563
|
@stats
|
504
564
|
end
|
505
565
|
|
506
566
|
private
|
507
567
|
|
568
|
+
def object_by_id_internal(id)
|
569
|
+
if (ruby_object_id = @in_memory_objects[id])
|
570
|
+
# We have the object in memory so we can just return it.
|
571
|
+
begin
|
572
|
+
object = ObjectSpace._id2ref(ruby_object_id)
|
573
|
+
# Let's make sure the object is really the object we are looking
|
574
|
+
# for. The GC might have recycled it already and the Ruby object ID
|
575
|
+
# could now be used for another object.
|
576
|
+
if object.is_a?(ObjectBase) && object._id == id
|
577
|
+
return object
|
578
|
+
end
|
579
|
+
rescue RangeError => e
|
580
|
+
# Due to a race condition the object can still be in the
|
581
|
+
# @in_memory_objects list but has been collected already by the Ruby
|
582
|
+
# GC. The _collect() call has not been completed yet. We now have to
|
583
|
+
# wait until this has been done. I think the GC lock will prevent a
|
584
|
+
# race on @in_memory_objects.
|
585
|
+
GC.start
|
586
|
+
while @in_memory_objects.include?(id)
|
587
|
+
sleep 0.01
|
588
|
+
end
|
589
|
+
end
|
590
|
+
end
|
591
|
+
|
592
|
+
# This is just a safety check. It has never triggered, so we can disable
|
593
|
+
# it for now.
|
594
|
+
#if (obj = @cache.object_by_id(id))
|
595
|
+
# PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
|
596
|
+
# "cache but not in_memory"
|
597
|
+
#end
|
598
|
+
|
599
|
+
# We don't have the object in memory. Let's find it in the storage.
|
600
|
+
if @db.include?(id)
|
601
|
+
# Great, object found. Read it into memory and return it.
|
602
|
+
obj = ObjectBase::read(self, id)
|
603
|
+
# Add the object to the in-memory storage list.
|
604
|
+
@cache.cache_read(obj)
|
605
|
+
|
606
|
+
return obj
|
607
|
+
end
|
608
|
+
|
609
|
+
# The requested object does not exist. Return nil.
|
610
|
+
nil
|
611
|
+
end
|
612
|
+
|
508
613
|
# Mark phase of a mark-and-sweep garbage collector. It will mark all
|
509
614
|
# objects that are reachable from the root objects.
|
510
615
|
def mark
|
511
616
|
classes = Set.new
|
512
617
|
marked_objects = 0
|
513
|
-
|
618
|
+
@progressmeter.start("Marking linked objects", @db.item_counter) do
|
619
|
+
each do |obj|
|
620
|
+
classes.add(obj.class)
|
621
|
+
@progressmeter.update(marked_objects += 1)
|
622
|
+
end
|
623
|
+
end
|
514
624
|
@class_map.keep(classes.map { |c| c.to_s })
|
515
625
|
|
516
626
|
# The root_objects object is included in the count, but we only want to
|
517
627
|
# count user objects here.
|
518
|
-
PEROBS.log.debug "#{marked_objects - 1}
|
628
|
+
PEROBS.log.debug "#{marked_objects - 1} of #{@db.item_counter} " +
|
629
|
+
"objects marked"
|
519
630
|
@stats.marked_objects = marked_objects - 1
|
520
631
|
end
|
521
632
|
|
522
633
|
# Sweep phase of a mark-and-sweep garbage collector. It will remove all
|
523
634
|
# unmarked objects from the store.
|
524
635
|
def sweep
|
525
|
-
@stats.swept_objects = @db.delete_unmarked_objects
|
526
|
-
|
636
|
+
@stats.swept_objects = @db.delete_unmarked_objects do |id|
|
637
|
+
@cache.evict(id)
|
638
|
+
end
|
639
|
+
@db.clear_marks
|
640
|
+
GC.start
|
527
641
|
PEROBS.log.debug "#{@stats.swept_objects} objects collected"
|
528
642
|
@stats.swept_objects
|
529
643
|
end
|
@@ -534,8 +648,7 @@ module PEROBS
|
|
534
648
|
# with
|
535
649
|
# @param repair [Boolean] Delete refernces to broken objects if true
|
536
650
|
# @return [Integer] The number of references to bad objects.
|
537
|
-
def check_object(start_id, repair)
|
538
|
-
errors = 0
|
651
|
+
def check_object(start_id, repair, stats)
|
539
652
|
@db.mark(start_id)
|
540
653
|
# The todo list holds a touple for each object that still needs to be
|
541
654
|
# checked. The first item is the referring object and the second is the
|
@@ -546,7 +659,13 @@ module PEROBS
|
|
546
659
|
# Get the next PEROBS object to check
|
547
660
|
ref_obj, id = todo_list.pop
|
548
661
|
|
549
|
-
|
662
|
+
begin
|
663
|
+
obj = object_by_id(id)
|
664
|
+
rescue PEROBS::FatalError
|
665
|
+
obj = nil
|
666
|
+
end
|
667
|
+
|
668
|
+
if obj
|
550
669
|
# The object exists and is OK. Mark is as checked.
|
551
670
|
@db.mark(id)
|
552
671
|
# Now look at all other objects referenced by this object.
|
@@ -569,11 +688,11 @@ module PEROBS
|
|
569
688
|
ref_obj.inspect
|
570
689
|
end
|
571
690
|
end
|
572
|
-
errors += 1
|
691
|
+
stats[:errors] += 1
|
573
692
|
end
|
574
|
-
end
|
575
693
|
|
576
|
-
|
694
|
+
@progressmeter.update(stats[:object_cnt] += 1)
|
695
|
+
end
|
577
696
|
end
|
578
697
|
|
579
698
|
end
|
data/lib/perobs/version.rb
CHANGED