perobs 4.0.0 → 4.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -16
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +106 -15
  5. data/lib/perobs/BTreeBlob.rb +4 -3
  6. data/lib/perobs/BTreeDB.rb +5 -4
  7. data/lib/perobs/BTreeNode.rb +482 -156
  8. data/lib/perobs/BTreeNodeLink.rb +10 -0
  9. data/lib/perobs/BigArray.rb +285 -0
  10. data/lib/perobs/BigArrayNode.rb +1002 -0
  11. data/lib/perobs/BigHash.rb +246 -0
  12. data/lib/perobs/BigTree.rb +197 -0
  13. data/lib/perobs/BigTreeNode.rb +873 -0
  14. data/lib/perobs/Cache.rb +48 -10
  15. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  16. data/lib/perobs/DataBase.rb +4 -3
  17. data/lib/perobs/DynamoDB.rb +57 -15
  18. data/lib/perobs/EquiBlobsFile.rb +155 -50
  19. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  20. data/lib/perobs/FlatFile.rb +519 -227
  21. data/lib/perobs/FlatFileBlobHeader.rb +113 -54
  22. data/lib/perobs/FlatFileDB.rb +49 -23
  23. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  24. data/lib/perobs/Hash.rb +127 -33
  25. data/lib/perobs/IDList.rb +144 -0
  26. data/lib/perobs/IDListPage.rb +107 -0
  27. data/lib/perobs/IDListPageFile.rb +180 -0
  28. data/lib/perobs/IDListPageRecord.rb +142 -0
  29. data/lib/perobs/Object.rb +18 -15
  30. data/lib/perobs/ObjectBase.rb +46 -5
  31. data/lib/perobs/PersistentObjectCache.rb +57 -68
  32. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  33. data/lib/perobs/ProgressMeter.rb +97 -0
  34. data/lib/perobs/SpaceManager.rb +273 -0
  35. data/lib/perobs/SpaceTree.rb +21 -12
  36. data/lib/perobs/SpaceTreeNode.rb +53 -61
  37. data/lib/perobs/Store.rb +264 -145
  38. data/lib/perobs/version.rb +1 -1
  39. data/lib/perobs.rb +2 -0
  40. data/perobs.gemspec +4 -4
  41. data/test/Array_spec.rb +15 -6
  42. data/test/BTree_spec.rb +6 -2
  43. data/test/BigArray_spec.rb +261 -0
  44. data/test/BigHash_spec.rb +152 -0
  45. data/test/BigTreeNode_spec.rb +153 -0
  46. data/test/BigTree_spec.rb +259 -0
  47. data/test/EquiBlobsFile_spec.rb +105 -1
  48. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  49. data/test/FlatFileDB_spec.rb +198 -14
  50. data/test/FuzzyStringMatcher_spec.rb +261 -0
  51. data/test/Hash_spec.rb +13 -3
  52. data/test/IDList_spec.rb +77 -0
  53. data/test/LegacyDBs/LegacyDB.rb +155 -0
  54. data/test/LegacyDBs/version_3/class_map.json +1 -0
  55. data/test/LegacyDBs/version_3/config.json +1 -0
  56. data/test/LegacyDBs/version_3/database.blobs +0 -0
  57. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  58. data/test/LegacyDBs/version_3/index.blobs +0 -0
  59. data/test/LegacyDBs/version_3/version +1 -0
  60. data/test/LockFile_spec.rb +9 -6
  61. data/test/SpaceManager_spec.rb +176 -0
  62. data/test/SpaceTree_spec.rb +4 -1
  63. data/test/Store_spec.rb +305 -203
  64. data/test/spec_helper.rb +9 -4
  65. metadata +57 -16
  66. data/lib/perobs/BTreeNodeCache.rb +0 -109
  67. data/lib/perobs/TreeDB.rb +0 -277
data/lib/perobs/Store.rb CHANGED
@@ -2,7 +2,8 @@
2
2
  #
3
3
  # = Store.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
6
+ # by Chris Schlaeger <chris@taskjuggler.org>
6
7
  #
7
8
  # MIT License
8
9
  #
@@ -26,6 +27,7 @@
26
27
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28
 
28
29
  require 'set'
30
+ require 'monitor'
29
31
 
30
32
  require 'perobs/Log'
31
33
  require 'perobs/Handle'
@@ -36,12 +38,18 @@ require 'perobs/FlatFileDB'
36
38
  require 'perobs/Object'
37
39
  require 'perobs/Hash'
38
40
  require 'perobs/Array'
41
+ require 'perobs/BigTree'
42
+ require 'perobs/BigHash'
43
+ require 'perobs/BigArray'
44
+ require 'perobs/ProgressMeter'
45
+ require 'perobs/ConsoleProgressMeter'
39
46
 
40
47
  # PErsistent Ruby OBject Store
41
48
  module PEROBS
42
49
 
43
50
  Statistics = Struct.new(:in_memory_objects, :root_objects,
44
- :marked_objects, :swept_objects)
51
+ :marked_objects, :swept_objects,
52
+ :created_objects, :collected_objects)
45
53
 
46
54
  # PEROBS::Store is a persistent storage system for Ruby objects. Regular
47
55
  # Ruby objects are transparently stored in a back-end storage and retrieved
@@ -103,6 +111,7 @@ module PEROBS
103
111
  class Store
104
112
 
105
113
  attr_reader :db, :cache, :class_map
114
+ attr_writer :root_objects
106
115
 
107
116
  # Create a new Store.
108
117
  # @param data_base [String] the name of the database
@@ -132,8 +141,17 @@ module PEROBS
132
141
  # :yaml : Can also handle most Ruby data types and is
133
142
  # portable between Ruby versions (1.9 and later).
134
143
  # Unfortunately, it is 10x slower than marshal.
144
+ # :progressmeter : reference to a ProgressMeter object that receives
145
+ # progress information during longer running tasks.
146
+ # It defaults to ProgressMeter which only logs into
147
+ # the log. Use ConsoleProgressMeter or a derived
148
+ # class for more fancy progress reporting.
149
+ # :no_root_objects : Create a new store without root objects. This only
150
+ # makes sense if you want to copy the objects of
151
+ # another store into this store.
135
152
  def initialize(data_base, options = {})
136
153
  # Create a backing store handler
154
+ @progressmeter = (options[:progressmeter] ||= ProgressMeter.new)
137
155
  @db = (options[:engine] || FlatFileDB).new(data_base, options)
138
156
  @db.open
139
157
  # Create a map that can translate classes to numerical IDs and vice
@@ -146,22 +164,29 @@ module PEROBS
146
164
 
147
165
  # This objects keeps some counters of interest.
148
166
  @stats = Statistics.new
167
+ @stats[:created_objects] = 0
168
+ @stats[:collected_objects] = 0
149
169
 
150
170
  # The Cache reduces read and write latencies by keeping a subset of the
151
171
  # objects in memory.
152
172
  @cache = Cache.new(options[:cache_bits] || 16)
153
173
 
174
+ # Lock to serialize access to the Store and all stored data.
175
+ @lock = Monitor.new
176
+
154
177
  # The named (global) objects IDs hashed by their name
155
- unless (@root_objects = object_by_id(0))
156
- PEROBS.log.debug "Initializing the PEROBS store"
157
- # The root object hash always has the object ID 0.
158
- @root_objects = _construct_po(Hash, 0)
159
- # Mark the root_objects object as modified.
160
- @cache.cache_write(@root_objects)
161
- end
162
- unless @root_objects.is_a?(Hash)
163
- PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
164
- "but is a #{@root_objects.class}"
178
+ unless options[:no_root_objects]
179
+ unless (@root_objects = object_by_id(0))
180
+ PEROBS.log.debug "Initializing the PEROBS store"
181
+ # The root object hash always has the object ID 0.
182
+ @root_objects = _construct_po(Hash, 0)
183
+ # Mark the root_objects object as modified.
184
+ @cache.cache_write(@root_objects)
185
+ end
186
+ unless @root_objects.is_a?(Hash)
187
+ PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
188
+ "but is a #{@root_objects.class}"
189
+ end
165
190
  end
166
191
  end
167
192
 
@@ -173,7 +198,9 @@ module PEROBS
173
198
  sync
174
199
 
175
200
  # Create a new store with the specified directory and options.
176
- new_db = Store.new(dir, options)
201
+ new_options = options.clone
202
+ new_options[:no_root_objects] = true
203
+ new_db = Store.new(dir, new_options)
177
204
  # Clear the cache.
178
205
  new_db.sync
179
206
  # Copy all objects of the existing store to the new store.
@@ -184,6 +211,7 @@ module PEROBS
184
211
  obj._sync
185
212
  i += 1
186
213
  end
214
+ new_db.root_objects = new_db.object_by_id(0)
187
215
  PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
188
216
  # Flush the new store and close it.
189
217
  new_db.exit
@@ -191,20 +219,34 @@ module PEROBS
191
219
  true
192
220
  end
193
221
 
194
-
195
222
  # Close the store and ensure that all in-memory objects are written out to
196
223
  # the storage backend. The Store object is no longer usable after this
197
224
  # method was called.
198
225
  def exit
199
226
  if @cache && @cache.in_transaction?
200
- PEROBS.log.fatal 'You cannot call exit() during a transaction'
227
+ @cache.abort_transaction
228
+ @cache.flush
229
+ @db.close if @db
230
+ PEROBS.log.fatal "You cannot call exit() during a transaction: #{Kernel.caller}"
201
231
  end
202
232
  @cache.flush if @cache
203
233
  @db.close if @db
204
- @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
205
- nil
206
- end
207
234
 
235
+ GC.start
236
+ if @stats
237
+ unless @stats[:created_objects] == @stats[:collected_objects] +
238
+ @in_memory_objects.length
239
+ PEROGS.log.fatal "Created objects count " +
240
+ "(#{@stats[:created_objects]})" +
241
+ " is not equal to the collected count " +
242
+ "(#{@stats[:collected_objects]}) + in_memory_objects count " +
243
+ "(#{@in_memory_objects.length})"
244
+ end
245
+ end
246
+
247
+ @db = @class_map = @in_memory_objects = @stats = @cache =
248
+ @root_objects = nil
249
+ end
208
250
 
209
251
  # You need to call this method to create new PEROBS objects that belong to
210
252
  # this Store.
@@ -218,11 +260,13 @@ module PEROBS
218
260
  PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
219
261
  end
220
262
 
221
- obj = _construct_po(klass, _new_id, *args)
222
- # Mark the new object as modified so it gets pushed into the database.
223
- @cache.cache_write(obj)
224
- # Return a POXReference proxy for the newly created object.
225
- obj.myself
263
+ @lock.synchronize do
264
+ obj = _construct_po(klass, _new_id, *args)
265
+ # Mark the new object as modified so it gets pushed into the database.
266
+ @cache.cache_write(obj)
267
+ # Return a POXReference proxy for the newly created object.
268
+ obj.myself
269
+ end
226
270
  end
227
271
 
228
272
  # For library internal use only!
@@ -239,9 +283,11 @@ module PEROBS
239
283
  # method was called. This is an alternative to exit() that additionaly
240
284
  # deletes the entire database.
241
285
  def delete_store
242
- @db.delete_database
243
- @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
244
- nil
286
+ @lock.synchronize do
287
+ @db.delete_database
288
+ @db = @class_map = @in_memory_objects = @stats = @cache =
289
+ @root_objects = nil
290
+ end
245
291
  end
246
292
 
247
293
  # Store the provided object under the given name. Use this to make the
@@ -253,25 +299,27 @@ module PEROBS
253
299
  # @param obj [PEROBS::Object] The object to store
254
300
  # @return [PEROBS::Object] The stored object.
255
301
  def []=(name, obj)
256
- # If the passed object is nil, we delete the entry if it exists.
257
- if obj.nil?
258
- @root_objects.delete(name)
259
- return nil
260
- end
302
+ @lock.synchronize do
303
+ # If the passed object is nil, we delete the entry if it exists.
304
+ if obj.nil?
305
+ @root_objects.delete(name)
306
+ return nil
307
+ end
261
308
 
262
- # We only allow derivatives of PEROBS::Object to be stored in the
263
- # store.
264
- unless obj.is_a?(ObjectBase)
265
- PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
266
- "is of class #{obj.class}"
267
- end
309
+ # We only allow derivatives of PEROBS::Object to be stored in the
310
+ # store.
311
+ unless obj.is_a?(ObjectBase)
312
+ PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
313
+ "is of class #{obj.class}"
314
+ end
268
315
 
269
- unless obj.store == self
270
- PEROBS.log.fatal 'The object does not belong to this store.'
271
- end
316
+ unless obj.store == self
317
+ PEROBS.log.fatal 'The object does not belong to this store.'
318
+ end
272
319
 
273
- # Store the name and mark the name list as modified.
274
- @root_objects[name] = obj._id
320
+ # Store the name and mark the name list as modified.
321
+ @root_objects[name] = obj._id
322
+ end
275
323
 
276
324
  obj
277
325
  end
@@ -281,25 +329,46 @@ module PEROBS
281
329
  # returned.
282
330
  # @return The requested object or nil if it doesn't exist.
283
331
  def [](name)
284
- # Return nil if there is no object with that name.
285
- return nil unless (id = @root_objects[name])
332
+ @lock.synchronize do
333
+ # Return nil if there is no object with that name.
334
+ return nil unless (id = @root_objects[name])
286
335
 
287
- POXReference.new(self, id)
336
+ POXReference.new(self, id)
337
+ end
288
338
  end
289
339
 
290
340
  # Return a list with all the names of the root objects.
291
341
  # @return [Array of Symbols]
292
342
  def names
293
- @root_objects.keys
343
+ @lock.synchronize do
344
+ @root_objects.keys
345
+ end
294
346
  end
295
347
 
296
348
  # Flush out all modified objects to disk and shrink the in-memory list if
297
349
  # needed.
298
350
  def sync
299
- if @cache.in_transaction?
300
- PEROBS.log.fatal 'You cannot call sync() during a transaction'
351
+ @lock.synchronize do
352
+ if @cache.in_transaction?
353
+ @cache.abort_transaction
354
+ @cache.flush
355
+ PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
356
+ Kernel.caller.join("\n")
357
+ end
358
+ @cache.flush
359
+ end
360
+ end
361
+
362
+ # Return the number of object stored in the store. CAVEAT: This method
363
+ # will only return correct values when it is separated from any mutating
364
+ # call by a call to sync().
365
+ # @return [Integer] Number of persistently stored objects in the Store.
366
+ def size
367
+ # We don't include the Hash that stores the root objects into the object
368
+ # count.
369
+ @lock.synchronize do
370
+ @db.item_counter - 1
301
371
  end
302
- @cache.flush
303
372
  end
304
373
 
305
374
  # Discard all objects that are not somehow connected to the root objects
@@ -308,58 +377,20 @@ module PEROBS
308
377
  # method periodically.
309
378
  # @return [Integer] The number of collected objects
310
379
  def gc
311
- if @cache.in_transaction?
312
- PEROBS.log.fatal 'You cannot call gc() during a transaction'
380
+ @lock.synchronize do
381
+ sync
382
+ mark
383
+ sweep
313
384
  end
314
- sync
315
- mark
316
- sweep
317
385
  end
318
386
 
319
387
  # Return the object with the provided ID. This method is not part of the
320
388
  # public API and should never be called by outside users. It's purely
321
389
  # intended for internal use.
322
390
  def object_by_id(id)
323
- if (ruby_object_id = @in_memory_objects[id])
324
- # We have the object in memory so we can just return it.
325
- begin
326
- object = ObjectSpace._id2ref(ruby_object_id)
327
- # Let's make sure the object is really the object we are looking
328
- # for. The GC might have recycled it already and the Ruby object ID
329
- # could now be used for another object.
330
- if object.is_a?(ObjectBase) && object._id == id
331
- return object
332
- end
333
- rescue RangeError => e
334
- # Due to a race condition the object can still be in the
335
- # @in_memory_objects list but has been collected already by the Ruby
336
- # GC. In that case we need to load it again. In this case the
337
- # _collect() call will happen much later, potentially after we have
338
- # registered a new object with the same ID.
339
- @in_memory_objects.delete(id)
340
- end
391
+ @lock.synchronize do
392
+ object_by_id_internal(id)
341
393
  end
342
-
343
- if (obj = @cache.object_by_id(id))
344
- PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
345
- end
346
-
347
- # We don't have the object in memory. Let's find it in the storage.
348
- if @db.include?(id)
349
- # Great, object found. Read it into memory and return it.
350
- obj = ObjectBase::read(self, id)
351
- # Add the object to the in-memory storage list.
352
- @cache.cache_read(obj)
353
-
354
- return obj
355
- end
356
-
357
- #if (obj = @db.search_object(id))
358
- # PEROBS.log.fatal "Object was not in index but in DB"
359
- #end
360
-
361
- # The requested object does not exist. Return nil.
362
- nil
363
394
  end
364
395
 
365
396
  # This method can be used to check the database and optionally repair it.
@@ -370,38 +401,42 @@ module PEROBS
370
401
  # made.
371
402
  # @return [Integer] The number of references to bad objects found.
372
403
  def check(repair = false)
404
+ stats = { :errors => 0, :object_cnt => 0 }
405
+
373
406
  # All objects must have in-db version.
374
407
  sync
375
408
  # Run basic consistency checks first.
376
- errors = @db.check_db(repair)
409
+ stats[:errors] += @db.check_db(repair)
377
410
 
378
411
  # We will use the mark to mark all objects that we have checked already.
379
412
  # Before we start, we need to clear all marks.
380
413
  @db.clear_marks
381
414
 
382
- objects = 0
383
- @root_objects.each do |name, id|
384
- objects += 1
385
- errors += check_object(id, repair)
415
+ @progressmeter.start("Checking object link structure",
416
+ @db.item_counter) do
417
+ @root_objects.each do |name, id|
418
+ check_object(id, repair, stats)
419
+ end
386
420
  end
387
421
 
388
422
  # Delete all broken root objects.
389
423
  if repair
390
424
  @root_objects.delete_if do |name, id|
391
- unless (res = @db.check(id, repair))
425
+ unless @db.check(id, repair)
392
426
  PEROBS.log.error "Discarding broken root object '#{name}' " +
393
427
  "with ID #{id}"
394
- errors += 1
428
+ stats[:errors] += 1
395
429
  end
396
- !res
397
430
  end
398
431
  end
399
432
 
400
- if errors > 0
433
+ if stats[:errors] > 0
401
434
  if repair
402
- PEROBS.log.error "#{errors} errors found in #{objects} objects"
435
+ PEROBS.log.error "#{stats[:errors]} errors found in " +
436
+ "#{stats[:object_cnt]} objects"
403
437
  else
404
- PEROBS.log.fatal "#{errors} errors found in #{objects} objects"
438
+ PEROBS.log.fatal "#{stats[:errors]} errors found in " +
439
+ "#{stats[:object_cnt]} objects"
405
440
  end
406
441
  else
407
442
  PEROBS.log.debug "No errors found"
@@ -410,7 +445,7 @@ module PEROBS
410
445
  # Ensure that any fixes are written into the DB.
411
446
  sync if repair
412
447
 
413
- errors
448
+ stats[:errors]
414
449
  end
415
450
 
416
451
  # This method will execute the provided block as an atomic transaction
@@ -420,35 +455,40 @@ module PEROBS
420
455
  # beginning of the transaction. The exception is passed on to the
421
456
  # enclosing scope, so you probably want to handle it accordingly.
422
457
  def transaction
423
- @cache.begin_transaction
458
+ @lock.synchronize { @cache.begin_transaction }
424
459
  begin
425
460
  yield if block_given?
426
461
  rescue => e
427
- @cache.abort_transaction
462
+ @lock.synchronize { @cache.abort_transaction }
428
463
  raise e
429
464
  end
430
- @cache.end_transaction
465
+ @lock.synchronize { @cache.end_transaction }
431
466
  end
432
467
 
433
468
  # Calls the given block once for each object, passing that object as a
434
469
  # parameter.
435
470
  def each
436
- @db.clear_marks
437
- # Start with the object 0 and the indexes of the root objects. Push them
438
- # onto the work stack.
439
- stack = [ 0 ] + @root_objects.values
440
- while !stack.empty?
441
- # Get an object index from the stack.
442
- unless (obj = object_by_id(id = stack.pop))
443
- PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
444
- "not found."
445
- end
446
- # Mark the object so it will never be pushed to the stack again.
447
- @db.mark(id)
448
- yield(obj.myself) if block_given?
449
- # Push the IDs of all unmarked referenced objects onto the stack
450
- obj._referenced_object_ids.each do |r_id|
451
- stack << r_id unless @db.is_marked?(r_id)
471
+ @lock.synchronize do
472
+ @db.clear_marks
473
+ # Start with the object 0 and the indexes of the root objects. Push them
474
+ # onto the work stack.
475
+ stack = [ 0 ] + @root_objects.values
476
+ while !stack.empty?
477
+ # Get an object index from the stack.
478
+ id = stack.pop
479
+ next if @db.is_marked?(id)
480
+
481
+ unless (obj = object_by_id_internal(id))
482
+ PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
483
+ "not found."
484
+ end
485
+ # Mark the object so it will never be pushed to the stack again.
486
+ @db.mark(id)
487
+ yield(obj.myself) if block_given?
488
+ # Push the IDs of all unmarked referenced objects onto the stack
489
+ obj._referenced_object_ids.each do |r_id|
490
+ stack << r_id unless @db.is_marked?(r_id)
491
+ end
452
492
  end
453
493
  end
454
494
  end
@@ -456,7 +496,7 @@ module PEROBS
456
496
  # Rename classes of objects stored in the data base.
457
497
  # @param rename_map [Hash] Hash that maps the old name to the new name
458
498
  def rename_classes(rename_map)
459
- @class_map.rename(rename_map)
499
+ @lock.synchronize { @class_map.rename(rename_map) }
460
500
  end
461
501
 
462
502
  # Internal method. Don't use this outside of this library!
@@ -464,14 +504,16 @@ module PEROBS
464
504
  # random numbers between 0 and 2**64 - 1.
465
505
  # @return [Integer]
466
506
  def _new_id
467
- begin
468
- # Generate a random number. It's recommended to not store more than
469
- # 2**62 objects in the same store.
470
- id = rand(2**64)
471
- # Ensure that we don't have already another object with this ID.
472
- end while @in_memory_objects.include?(id) || @db.include?(id)
507
+ @lock.synchronize do
508
+ begin
509
+ # Generate a random number. It's recommended to not store more than
510
+ # 2**62 objects in the same store.
511
+ id = rand(2**64)
512
+ # Ensure that we don't have already another object with this ID.
513
+ end while @in_memory_objects.include?(id) || @db.include?(id)
473
514
 
474
- id
515
+ id
516
+ end
475
517
  end
476
518
 
477
519
  # Internal method. Don't use this outside of this library!
@@ -482,7 +524,18 @@ module PEROBS
482
524
  # @param obj [BasicObject] Object to register
483
525
  # @param id [Integer] object ID
484
526
  def _register_in_memory(obj, id)
485
- @in_memory_objects[id] = obj.object_id
527
+ @lock.synchronize do
528
+ unless obj.is_a?(ObjectBase)
529
+ PEROBS.log.fatal "You can only register ObjectBase objects"
530
+ end
531
+ if @in_memory_objects.include?(id)
532
+ PEROBS.log.fatal "The Store::_in_memory_objects list already " +
533
+ "contains an object for ID #{id}"
534
+ end
535
+
536
+ @in_memory_objects[id] = obj.object_id
537
+ @stats[:created_objects] += 1
538
+ end
486
539
  end
487
540
 
488
541
  # Remove the object from the in-memory list. This is an internal method
@@ -490,40 +543,101 @@ module PEROBS
490
543
  # finalizer, so many restrictions apply!
491
544
  # @param id [Integer] Object ID of object to remove from the list
492
545
  def _collect(id, ruby_object_id)
546
+ # This method should only be called from the Ruby garbage collector.
547
+ # Therefor no locking is needed or even possible. The GC can kick in at
548
+ # any time and we could be anywhere in the code. So there is a small
549
+ # risk for a race here, but it should not have any serious consequences.
493
550
  if @in_memory_objects[id] == ruby_object_id
494
551
  @in_memory_objects.delete(id)
552
+ @stats[:collected_objects] += 1
495
553
  end
496
554
  end
497
555
 
498
556
  # This method returns a Hash with some statistics about this store.
499
557
  def statistics
500
- @stats.in_memory_objects = @in_memory_objects.length
501
- @stats.root_objects = @root_objects.length
558
+ @lock.synchronize do
559
+ @stats.in_memory_objects = @in_memory_objects.length
560
+ @stats.root_objects = @root_objects.length
561
+ end
502
562
 
503
563
  @stats
504
564
  end
505
565
 
506
566
  private
507
567
 
568
+ def object_by_id_internal(id)
569
+ if (ruby_object_id = @in_memory_objects[id])
570
+ # We have the object in memory so we can just return it.
571
+ begin
572
+ object = ObjectSpace._id2ref(ruby_object_id)
573
+ # Let's make sure the object is really the object we are looking
574
+ # for. The GC might have recycled it already and the Ruby object ID
575
+ # could now be used for another object.
576
+ if object.is_a?(ObjectBase) && object._id == id
577
+ return object
578
+ end
579
+ rescue RangeError => e
580
+ # Due to a race condition the object can still be in the
581
+ # @in_memory_objects list but has been collected already by the Ruby
582
+ # GC. The _collect() call has not been completed yet. We now have to
583
+ # wait until this has been done. I think the GC lock will prevent a
584
+ # race on @in_memory_objects.
585
+ GC.start
586
+ while @in_memory_objects.include?(id)
587
+ sleep 0.01
588
+ end
589
+ end
590
+ end
591
+
592
+ # This is just a safety check. It has never triggered, so we can disable
593
+ # it for now.
594
+ #if (obj = @cache.object_by_id(id))
595
+ # PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
596
+ # "cache but not in_memory"
597
+ #end
598
+
599
+ # We don't have the object in memory. Let's find it in the storage.
600
+ if @db.include?(id)
601
+ # Great, object found. Read it into memory and return it.
602
+ obj = ObjectBase::read(self, id)
603
+ # Add the object to the in-memory storage list.
604
+ @cache.cache_read(obj)
605
+
606
+ return obj
607
+ end
608
+
609
+ # The requested object does not exist. Return nil.
610
+ nil
611
+ end
612
+
508
613
  # Mark phase of a mark-and-sweep garbage collector. It will mark all
509
614
  # objects that are reachable from the root objects.
510
615
  def mark
511
616
  classes = Set.new
512
617
  marked_objects = 0
513
- each { |obj| classes.add(obj.class); marked_objects += 1 }
618
+ @progressmeter.start("Marking linked objects", @db.item_counter) do
619
+ each do |obj|
620
+ classes.add(obj.class)
621
+ @progressmeter.update(marked_objects += 1)
622
+ end
623
+ end
514
624
  @class_map.keep(classes.map { |c| c.to_s })
515
625
 
516
626
  # The root_objects object is included in the count, but we only want to
517
627
  # count user objects here.
518
- PEROBS.log.debug "#{marked_objects - 1} objects marked"
628
+ PEROBS.log.debug "#{marked_objects - 1} of #{@db.item_counter} " +
629
+ "objects marked"
519
630
  @stats.marked_objects = marked_objects - 1
520
631
  end
521
632
 
522
633
  # Sweep phase of a mark-and-sweep garbage collector. It will remove all
523
634
  # unmarked objects from the store.
524
635
  def sweep
525
- @stats.swept_objects = @db.delete_unmarked_objects.length
526
- @cache.reset
636
+ @stats.swept_objects = @db.delete_unmarked_objects do |id|
637
+ @cache.evict(id)
638
+ end
639
+ @db.clear_marks
640
+ GC.start
527
641
  PEROBS.log.debug "#{@stats.swept_objects} objects collected"
528
642
  @stats.swept_objects
529
643
  end
@@ -534,8 +648,7 @@ module PEROBS
534
648
  # with
535
649
  # @param repair [Boolean] Delete refernces to broken objects if true
536
650
  # @return [Integer] The number of references to bad objects.
537
- def check_object(start_id, repair)
538
- errors = 0
651
+ def check_object(start_id, repair, stats)
539
652
  @db.mark(start_id)
540
653
  # The todo list holds a touple for each object that still needs to be
541
654
  # checked. The first item is the referring object and the second is the
@@ -546,7 +659,13 @@ module PEROBS
546
659
  # Get the next PEROBS object to check
547
660
  ref_obj, id = todo_list.pop
548
661
 
549
- if (obj = object_by_id(id))
662
+ begin
663
+ obj = object_by_id(id)
664
+ rescue PEROBS::FatalError
665
+ obj = nil
666
+ end
667
+
668
+ if obj
550
669
  # The object exists and is OK. Mark is as checked.
551
670
  @db.mark(id)
552
671
  # Now look at all other objects referenced by this object.
@@ -569,11 +688,11 @@ module PEROBS
569
688
  ref_obj.inspect
570
689
  end
571
690
  end
572
- errors += 1
691
+ stats[:errors] += 1
573
692
  end
574
- end
575
693
 
576
- errors
694
+ @progressmeter.update(stats[:object_cnt] += 1)
695
+ end
577
696
  end
578
697
 
579
698
  end
@@ -1,4 +1,4 @@
1
1
  module PEROBS
2
2
  # The version number
3
- VERSION = "4.0.0"
3
+ VERSION = "4.4.0"
4
4
  end