perobs 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/perobs/Array.rb +17 -4
- data/lib/perobs/BTreeBlob.rb +22 -23
- data/lib/perobs/BTreeDB.rb +11 -12
- data/lib/perobs/Cache.rb +5 -4
- data/lib/perobs/DataBase.rb +19 -7
- data/lib/perobs/DynamoDB.rb +1 -1
- data/lib/perobs/FixedSizeBlobFile.rb +189 -0
- data/lib/perobs/FlatFile.rb +513 -0
- data/lib/perobs/FlatFileDB.rb +249 -0
- data/lib/perobs/FreeSpaceManager.rb +204 -0
- data/lib/perobs/Hash.rb +17 -4
- data/lib/perobs/IndexTree.rb +164 -0
- data/lib/perobs/IndexTreeNode.rb +296 -0
- data/lib/perobs/Log.rb +125 -0
- data/lib/perobs/Object.rb +10 -11
- data/lib/perobs/ObjectBase.rb +18 -5
- data/lib/perobs/StackFile.rb +137 -0
- data/lib/perobs/Store.rb +85 -19
- data/lib/perobs/TreeDB.rb +276 -0
- data/lib/perobs/version.rb +1 -1
- data/test/Array_spec.rb +11 -2
- data/test/FixedSizeBlobFile_spec.rb +91 -0
- data/test/FlatFileDB_spec.rb +56 -0
- data/test/FreeSpaceManager_spec.rb +91 -0
- data/test/Hash_spec.rb +11 -2
- data/test/IndexTree_spec.rb +118 -0
- data/test/Object_spec.rb +29 -17
- data/test/StackFile_spec.rb +113 -0
- data/test/Store_spec.rb +37 -3
- metadata +22 -3
data/lib/perobs/Store.rb
CHANGED
@@ -28,10 +28,12 @@
|
|
28
28
|
require 'set'
|
29
29
|
require 'weakref'
|
30
30
|
|
31
|
+
require 'perobs/Log'
|
31
32
|
require 'perobs/Handle'
|
32
33
|
require 'perobs/Cache'
|
33
34
|
require 'perobs/ClassMap'
|
34
35
|
require 'perobs/BTreeDB'
|
36
|
+
require 'perobs/FlatFileDB'
|
35
37
|
require 'perobs/Object'
|
36
38
|
require 'perobs/Hash'
|
37
39
|
require 'perobs/Array'
|
@@ -100,9 +102,9 @@ module PEROBS
|
|
100
102
|
# @param options [Hash] various options to affect the operation of the
|
101
103
|
# database. Currently the following options are supported:
|
102
104
|
# :engine : The class that provides the back-end storage
|
103
|
-
# engine. By default
|
105
|
+
# engine. By default FlatFileDB is used. A user
|
104
106
|
# can provide it's own storage engine that must
|
105
|
-
# conform to the same API exposed by
|
107
|
+
# conform to the same API exposed by FlatFileDB.
|
106
108
|
# :cache_bits : the number of bits used for cache indexing. The
|
107
109
|
# cache will hold 2 to the power of bits number of
|
108
110
|
# objects. We have separate caches for reading and
|
@@ -126,6 +128,7 @@ module PEROBS
|
|
126
128
|
def initialize(data_base, options = {})
|
127
129
|
# Create a backing store handler
|
128
130
|
@db = (options[:engine] || BTreeDB).new(data_base, options)
|
131
|
+
@db.open
|
129
132
|
# Create a map that can translate classes to numerical IDs and vice
|
130
133
|
# versa.
|
131
134
|
@class_map = ClassMap.new(@db)
|
@@ -143,13 +146,60 @@ module PEROBS
|
|
143
146
|
|
144
147
|
# The named (global) objects IDs hashed by their name
|
145
148
|
unless (@root_objects = object_by_id(0))
|
149
|
+
PEROBS.log.debug "Initializing the PEROBS store"
|
146
150
|
# The root object hash always has the object ID 0.
|
147
151
|
@root_objects = _construct_po(Hash, 0)
|
148
152
|
# Mark the root_objects object as modified.
|
149
153
|
@cache.cache_write(@root_objects)
|
150
154
|
end
|
155
|
+
unless @root_objects.is_a?(Hash)
|
156
|
+
PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
|
157
|
+
"but is a #{@root_objects.class}"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
# Copy the store content into a new Store. The arguments are identical to
|
162
|
+
# Store.new().
|
163
|
+
# @param data_base [String] the name of the database
|
164
|
+
# @param options [Hash] various options to affect the operation of the
|
165
|
+
def copy(dir, options = {})
|
166
|
+
# Make sure all objects are persisted.
|
167
|
+
sync
|
168
|
+
|
169
|
+
# Create a new store with the specified directory and options.
|
170
|
+
new_db = Store.new(dir, options)
|
171
|
+
# Clear the cache.
|
172
|
+
new_db.sync
|
173
|
+
# Copy all objects of the existing store to the new store.
|
174
|
+
i = 0
|
175
|
+
each do |ref_obj|
|
176
|
+
obj = ref_obj._referenced_object
|
177
|
+
obj._transfer(new_db)
|
178
|
+
obj._sync
|
179
|
+
i += 1
|
180
|
+
end
|
181
|
+
PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
|
182
|
+
# Flush the new store and close it.
|
183
|
+
new_db.exit
|
184
|
+
|
185
|
+
true
|
186
|
+
end
|
187
|
+
|
188
|
+
|
189
|
+
# Close the store and ensure that all in-memory objects are written out to
|
190
|
+
# the storage backend. The Store object is no longer usable after this
|
191
|
+
# method was called.
|
192
|
+
def exit
|
193
|
+
if @cache.in_transaction?
|
194
|
+
PEROBS.log.fatal 'You cannot call exit() during a transaction'
|
195
|
+
end
|
196
|
+
@cache.flush
|
197
|
+
@db.close
|
198
|
+
@db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
|
199
|
+
nil
|
151
200
|
end
|
152
201
|
|
202
|
+
|
153
203
|
# You need to call this method to create new PEROBS objects that belong to
|
154
204
|
# this Store.
|
155
205
|
# @param klass [Class] The class of the object you want to create. This
|
@@ -159,7 +209,7 @@ module PEROBS
|
|
159
209
|
# @return [POXReference] A reference to the newly created object.
|
160
210
|
def new(klass, *args)
|
161
211
|
unless klass.is_a?(BasicObject)
|
162
|
-
|
212
|
+
PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
|
163
213
|
end
|
164
214
|
|
165
215
|
obj = _construct_po(klass, _new_id, *args)
|
@@ -204,12 +254,12 @@ module PEROBS
|
|
204
254
|
# We only allow derivatives of PEROBS::Object to be stored in the
|
205
255
|
# store.
|
206
256
|
unless obj.is_a?(ObjectBase)
|
207
|
-
|
208
|
-
|
257
|
+
PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
|
258
|
+
"is of class #{obj.class}"
|
209
259
|
end
|
210
260
|
|
211
261
|
unless obj.store == self
|
212
|
-
|
262
|
+
PEROBS.log.fatal 'The object does not belong to this store.'
|
213
263
|
end
|
214
264
|
|
215
265
|
# Store the name and mark the name list as modified.
|
@@ -229,11 +279,17 @@ module PEROBS
|
|
229
279
|
POXReference.new(self, id)
|
230
280
|
end
|
231
281
|
|
282
|
+
# Return a list with all the names of the root objects.
|
283
|
+
# @return [Array of Symbols]
|
284
|
+
def names
|
285
|
+
@root_objects.keys
|
286
|
+
end
|
287
|
+
|
232
288
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
233
289
|
# needed.
|
234
290
|
def sync
|
235
291
|
if @cache.in_transaction?
|
236
|
-
|
292
|
+
PEROBS.log.fatal 'You cannot call sync() during a transaction'
|
237
293
|
end
|
238
294
|
@cache.flush
|
239
295
|
end
|
@@ -245,7 +301,7 @@ module PEROBS
|
|
245
301
|
# @return [Fixnum] The number of collected objects
|
246
302
|
def gc
|
247
303
|
if @cache.in_transaction?
|
248
|
-
|
304
|
+
PEROBS.log.fatal 'You cannot call gc() during a transaction'
|
249
305
|
end
|
250
306
|
sync
|
251
307
|
mark
|
@@ -299,11 +355,21 @@ module PEROBS
|
|
299
355
|
@db.clear_marks
|
300
356
|
|
301
357
|
errors = 0
|
358
|
+
objects = 0
|
302
359
|
@root_objects.each do |name, id|
|
360
|
+
objects += 1
|
303
361
|
errors += check_object(id, repair)
|
304
362
|
end
|
363
|
+
if errors > 0
|
364
|
+
PEROBS.log.warn "#{errors} errors found in #{objects} objects"
|
365
|
+
else
|
366
|
+
PEROBS.log.debug "No errors found"
|
367
|
+
end
|
305
368
|
@root_objects.delete_if { |name, id| !@db.check(id, false) }
|
306
369
|
|
370
|
+
# Ensure that any fixes are written into the DB.
|
371
|
+
sync
|
372
|
+
|
307
373
|
errors
|
308
374
|
end
|
309
375
|
|
@@ -334,8 +400,8 @@ module PEROBS
|
|
334
400
|
while !stack.empty?
|
335
401
|
# Get an object index from the stack.
|
336
402
|
unless (obj = object_by_id(id = stack.pop))
|
337
|
-
|
338
|
-
|
403
|
+
PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
|
404
|
+
"not found."
|
339
405
|
end
|
340
406
|
# Mark the object so it will never be pushed to the stack again.
|
341
407
|
@db.mark(id)
|
@@ -352,6 +418,7 @@ module PEROBS
|
|
352
418
|
def rename_classes(rename_map)
|
353
419
|
@class_map.rename(rename_map)
|
354
420
|
end
|
421
|
+
|
355
422
|
# Internal method. Don't use this outside of this library!
|
356
423
|
# Generate a new unique ID that is not used by any other object. It uses
|
357
424
|
# random numbers between 0 and 2**64 - 1.
|
@@ -379,12 +446,10 @@ module PEROBS
|
|
379
446
|
end
|
380
447
|
|
381
448
|
# Remove the object from the in-memory list. This is an internal method
|
382
|
-
# and should never be called from user code.
|
449
|
+
# and should never be called from user code. It will be called from a
|
450
|
+
# finalizer, so many restrictions apply!
|
383
451
|
# @param id [Fixnum or Bignum] Object ID of object to remove from the list
|
384
452
|
def _collect(id, ignore_errors = false)
|
385
|
-
unless ignore_errors || @in_memory_objects.include?(id)
|
386
|
-
raise RuntimeError, "Object with id #{id} is currently not in memory"
|
387
|
-
end
|
388
453
|
@in_memory_objects.delete(id)
|
389
454
|
end
|
390
455
|
|
@@ -408,6 +473,7 @@ module PEROBS
|
|
408
473
|
|
409
474
|
# The root_objects object is included in the count, but we only want to
|
410
475
|
# count user objects here.
|
476
|
+
PEROBS.log.debug "#{marked_objects - 1} objects marked"
|
411
477
|
@stats.marked_objects = marked_objects - 1
|
412
478
|
end
|
413
479
|
|
@@ -416,6 +482,7 @@ module PEROBS
|
|
416
482
|
def sweep
|
417
483
|
@stats.swept_objects = @db.delete_unmarked_objects.length
|
418
484
|
@cache.reset
|
485
|
+
PEROBS.log.debug "#{@stats.swept_objects} objects collected"
|
419
486
|
@stats.swept_objects
|
420
487
|
end
|
421
488
|
|
@@ -449,13 +516,12 @@ module PEROBS
|
|
449
516
|
else
|
450
517
|
# Remove references to bad objects.
|
451
518
|
if ref_obj && repair
|
452
|
-
|
453
|
-
|
519
|
+
PEROBS.log.warn "Fixing broken reference to object #{id} in " +
|
520
|
+
"object #{ref_obj._id}:\n" + ref_obj.inspect
|
454
521
|
ref_obj._delete_reference_to_id(id)
|
455
522
|
else
|
456
|
-
|
457
|
-
|
458
|
-
ref_obj.inspect
|
523
|
+
PEROBS.log.fatal "The following object references a " +
|
524
|
+
"non-existing object #{id}:\n" + ref_obj.inspect
|
459
525
|
end
|
460
526
|
errors += 1
|
461
527
|
end
|
@@ -0,0 +1,276 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# = BTreeDB.rb -- Persistent Ruby Object Store
|
4
|
+
#
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
+
#
|
7
|
+
# MIT License
|
8
|
+
#
|
9
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
+
# a copy of this software and associated documentation files (the
|
11
|
+
# "Software"), to deal in the Software without restriction, including
|
12
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
+
# the following conditions:
|
16
|
+
#
|
17
|
+
# The above copyright notice and this permission notice shall be
|
18
|
+
# included in all copies or substantial portions of the Software.
|
19
|
+
#
|
20
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
+
|
28
|
+
require 'fileutils'
|
29
|
+
|
30
|
+
require 'perobs/Log'
|
31
|
+
require 'perobs/DataBase'
|
32
|
+
require 'perobs/BTreeBlob'
|
33
|
+
|
34
|
+
module PEROBS
|
35
|
+
|
36
|
+
# This class implements a BTree database using filesystem directories as
|
37
|
+
# nodes and blob files as leafs. The BTree grows with the number of stored
|
38
|
+
# entries. Each leaf node blob can hold a fixed number of entries. If more
|
39
|
+
# entries need to be stored, the blob is replaced by a node with multiple
|
40
|
+
# new leafs that store the entries of the previous node. The leafs are
|
41
|
+
# implemented by the BTreeBlob class.
|
42
|
+
class BTreeDB < DataBase
|
43
|
+
|
44
|
+
attr_reader :max_blob_size
|
45
|
+
|
46
|
+
# Create a new BTreeDB object.
|
47
|
+
# @param db_name [String] name of the DB directory
|
48
|
+
# @param options [Hash] options to customize the behavior. Currently only
|
49
|
+
# the following options are supported:
|
50
|
+
# :serializer : Can be :marshal, :json, :yaml
|
51
|
+
# :dir_bits : The number of bits to use for the BTree nodes.
|
52
|
+
# The value must be between 4 and 14. The larger
|
53
|
+
# the number the more back-end directories are
|
54
|
+
# being used. The default is 12 which results in
|
55
|
+
# 4096 directories per node.
|
56
|
+
# :max_blob_size : The maximum number of entries in the BTree leaf
|
57
|
+
# nodes. The insert/find/delete time grows
|
58
|
+
# linearly with the size.
|
59
|
+
def initialize(db_name, options = {})
|
60
|
+
super(options[:serializer] || :json)
|
61
|
+
|
62
|
+
@db_dir = db_name
|
63
|
+
# Create the database directory if it doesn't exist yet.
|
64
|
+
ensure_dir_exists(@db_dir)
|
65
|
+
|
66
|
+
# Read the existing DB config.
|
67
|
+
@config = get_hash('config')
|
68
|
+
check_option('serializer')
|
69
|
+
|
70
|
+
# Check and set @dir_bits, the number of bits used for each tree level.
|
71
|
+
@dir_bits = options[:dir_bits] || 12
|
72
|
+
if @dir_bits < 4 || @dir_bits > 14
|
73
|
+
PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
|
74
|
+
"and 12"
|
75
|
+
end
|
76
|
+
check_option('dir_bits')
|
77
|
+
|
78
|
+
@max_blob_size = options[:max_blob_size] || 32
|
79
|
+
if @max_blob_size < 4 || @max_blob_size > 128
|
80
|
+
PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
|
81
|
+
"between 4 and 128"
|
82
|
+
end
|
83
|
+
check_option('max_blob_size')
|
84
|
+
|
85
|
+
put_hash('config', @config)
|
86
|
+
|
87
|
+
# This format string is used to create the directory name.
|
88
|
+
@dir_format_string = "%0#{(@dir_bits / 4) +
|
89
|
+
(@dir_bits % 4 == 0 ? 0 : 1)}X"
|
90
|
+
# Bit mask to extract the dir_bits LSBs.
|
91
|
+
@dir_mask = 2 ** @dir_bits - 1
|
92
|
+
end
|
93
|
+
|
94
|
+
# Delete the entire database. The database is no longer usable after this
|
95
|
+
# method was called.
|
96
|
+
def delete_database
|
97
|
+
FileUtils.rm_rf(@db_dir)
|
98
|
+
end
|
99
|
+
|
100
|
+
def BTreeDB::delete_db(db_name)
|
101
|
+
FileUtils.rm_rf(db_name)
|
102
|
+
end
|
103
|
+
|
104
|
+
# Return true if the object with given ID exists
|
105
|
+
# @param id [Fixnum or Bignum]
|
106
|
+
def include?(id)
|
107
|
+
!(blob = find_blob(id)).nil? && !blob.find(id).nil?
|
108
|
+
end
|
109
|
+
|
110
|
+
# Store a simple Hash as a JSON encoded file into the DB directory.
|
111
|
+
# @param name [String] Name of the hash. Will be used as file name.
|
112
|
+
# @param hash [Hash] A Hash that maps String objects to strings or
|
113
|
+
# numbers.
|
114
|
+
def put_hash(name, hash)
|
115
|
+
file_name = File.join(@db_dir, name + '.json')
|
116
|
+
begin
|
117
|
+
File.write(file_name, hash.to_json)
|
118
|
+
rescue => e
|
119
|
+
PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
# Load the Hash with the given name.
|
124
|
+
# @param name [String] Name of the hash.
|
125
|
+
# @return [Hash] A Hash that maps String objects to strings or numbers.
|
126
|
+
def get_hash(name)
|
127
|
+
file_name = File.join(@db_dir, name + '.json')
|
128
|
+
return ::Hash.new unless File.exist?(file_name)
|
129
|
+
|
130
|
+
begin
|
131
|
+
json = File.read(file_name)
|
132
|
+
rescue => e
|
133
|
+
PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
|
134
|
+
end
|
135
|
+
JSON.parse(json, :create_additions => true)
|
136
|
+
end
|
137
|
+
|
138
|
+
# Store the given object into the cluster files.
|
139
|
+
# @param obj [Hash] Object as defined by PEROBS::ObjectBase
|
140
|
+
def put_object(obj, id)
|
141
|
+
find_blob(id, true).write_object(id, serialize(obj))
|
142
|
+
end
|
143
|
+
|
144
|
+
# Load the given object from the filesystem.
|
145
|
+
# @param id [Fixnum or Bignum] object ID
|
146
|
+
# @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
|
147
|
+
# not exist
|
148
|
+
def get_object(id)
|
149
|
+
return nil unless (blob = find_blob(id)) && (obj = blob.read_object(id))
|
150
|
+
deserialize(obj)
|
151
|
+
end
|
152
|
+
|
153
|
+
# This method must be called to initiate the marking process.
|
154
|
+
def clear_marks
|
155
|
+
each_blob { |blob| blob.clear_marks }
|
156
|
+
end
|
157
|
+
|
158
|
+
# Permanently delete all objects that have not been marked. Those are
|
159
|
+
# orphaned and are no longer referenced by any actively used object.
|
160
|
+
# @return [Array] List of IDs that have been removed from the DB.
|
161
|
+
def delete_unmarked_objects
|
162
|
+
deleted_ids = []
|
163
|
+
each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
|
164
|
+
deleted_ids
|
165
|
+
end
|
166
|
+
|
167
|
+
# Mark an object.
|
168
|
+
# @param id [Fixnum or Bignum] ID of the object to mark
|
169
|
+
def mark(id)
|
170
|
+
(blob = find_blob(id)) && blob.mark(id)
|
171
|
+
end
|
172
|
+
|
173
|
+
# Check if the object is marked.
|
174
|
+
# @param id [Fixnum or Bignum] ID of the object to check
|
175
|
+
# @param ignore_errors [Boolean] If set to true no errors will be raised
|
176
|
+
# for non-existing objects.
|
177
|
+
def is_marked?(id, ignore_errors = false)
|
178
|
+
(blob = find_blob(id)) && blob.is_marked?(id, ignore_errors)
|
179
|
+
end
|
180
|
+
|
181
|
+
# Basic consistency check.
|
182
|
+
# @param repair [TrueClass/FalseClass] True if found errors should be
|
183
|
+
# repaired.
|
184
|
+
def check_db(repair = false)
|
185
|
+
each_blob { |blob| blob.check(repair) }
|
186
|
+
end
|
187
|
+
|
188
|
+
# Check if the stored object is syntactically correct.
|
189
|
+
# @param id [Fixnum/Bignum] Object ID
|
190
|
+
# @param repair [TrueClass/FalseClass] True if an repair attempt should be
|
191
|
+
# made.
|
192
|
+
# @return [TrueClass/FalseClass] True if the object is OK, otherwise
|
193
|
+
# false.
|
194
|
+
def check(id, repair)
|
195
|
+
begin
|
196
|
+
get_object(id)
|
197
|
+
rescue => e
|
198
|
+
PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
|
199
|
+
return false
|
200
|
+
end
|
201
|
+
|
202
|
+
true
|
203
|
+
end
|
204
|
+
|
205
|
+
# Store the given serialized object into the cluster files. This method is
|
206
|
+
# for internal use only!
|
207
|
+
# @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
|
208
|
+
# @param id [Fixnum or Bignum] Object ID
|
209
|
+
def put_raw_object(raw, id)
|
210
|
+
find_blob(id, true).write_object(id, raw)
|
211
|
+
end
|
212
|
+
|
213
|
+
private
|
214
|
+
|
215
|
+
def find_blob(id, create_missing_blob = false, dir_name = @db_dir)
|
216
|
+
dir_bits = id & @dir_mask
|
217
|
+
sub_dir_name = File.join(dir_name, @dir_format_string % dir_bits)
|
218
|
+
|
219
|
+
if Dir.exist?(sub_dir_name)
|
220
|
+
if File.exist?(File.join(sub_dir_name, 'index'))
|
221
|
+
# The directory is a blob directory and not a BTree node dir.
|
222
|
+
return BTreeBlob.new(sub_dir_name, self)
|
223
|
+
end
|
224
|
+
else
|
225
|
+
Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
|
226
|
+
# Extract the 01-part of the filename
|
227
|
+
lsb_string = File.basename(fqfn)[0..-6]
|
228
|
+
# Convert the lsb_string into a Fixnum
|
229
|
+
lsb = Integer('0b' + lsb_string)
|
230
|
+
# Bit mask to match the LSBs
|
231
|
+
mask = (2 ** lsb_string.length) - 1
|
232
|
+
if (id & mask) == lsb
|
233
|
+
return TreeBlob.new(sub_dir_name, lsb_string, self)
|
234
|
+
end
|
235
|
+
end
|
236
|
+
if create_missing_blob
|
237
|
+
# Create the new blob directory.
|
238
|
+
Dir.mkdir(dir_name)
|
239
|
+
# And initialize the blob DB.
|
240
|
+
return BTreeBlob.new(dir_name, self)
|
241
|
+
else
|
242
|
+
return nil
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
# Discard the least significant @dir_bits bits and start over again
|
247
|
+
# with the directory that matches the @dir_bits LSBs of the new ID.
|
248
|
+
id = id >> @dir_bits
|
249
|
+
end
|
250
|
+
|
251
|
+
def each_blob(&block)
|
252
|
+
each_blob_r(@db_dir, &block)
|
253
|
+
end
|
254
|
+
|
255
|
+
def each_blob_r(dir, &block)
|
256
|
+
Dir.glob(File.join(dir, '*')) do |dir_name|
|
257
|
+
if is_blob_dir?(dir_name)
|
258
|
+
block.call(BTreeBlob.new(dir_name, self))
|
259
|
+
else
|
260
|
+
each_blob_r(dir_name, &block)
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def is_blob_dir?(dir_name)
|
266
|
+
# A blob directory contains an 'index' and 'data' file. This is in
|
267
|
+
# contrast to BTree node directories that only contain other
|
268
|
+
# directories.
|
269
|
+
index_file = File.join(dir_name, 'index')
|
270
|
+
File.exist?(index_file)
|
271
|
+
end
|
272
|
+
|
273
|
+
end
|
274
|
+
|
275
|
+
end
|
276
|
+
|
data/lib/perobs/version.rb
CHANGED
data/test/Array_spec.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
3
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
4
|
#
|
5
5
|
# This file contains tests for Array that are similar to the tests for the
|
6
6
|
# Array implementation in MRI. The ideas of these tests were replicated in
|
@@ -224,6 +224,13 @@ describe PEROBS::Array do
|
|
224
224
|
pcheck { expect(a).to eq([ 1, 2, 3, 4, 5, nil ]) }
|
225
225
|
end
|
226
226
|
|
227
|
+
it 'should support inspect' do
|
228
|
+
a1 = cpa([ 1 ])
|
229
|
+
a2 = cpa([ 1, a1 ])
|
230
|
+
expect(a1.inspect).to eq("<PEROBS::Array:#{a1._id}>\n[\n 1\n]\n")
|
231
|
+
expect(a2.inspect).to eq("<PEROBS::Array:#{a2._id}>\n[\n 1,\n <PEROBS::ObjectBase:#{a1._id}>\n]\n")
|
232
|
+
end
|
233
|
+
|
227
234
|
it 'should only provide POXReference objects' do
|
228
235
|
a = cpa([ @store.new(PO), @store.new(PO) ])
|
229
236
|
expect(a[0].respond_to?(:is_poxreference?)).to be true
|
@@ -236,7 +243,9 @@ describe PEROBS::Array do
|
|
236
243
|
@store['a'] = a = @store.new(PEROBS::Array)
|
237
244
|
o = @store.new(PO)
|
238
245
|
a[0] = o.get_self
|
239
|
-
|
246
|
+
PEROBS.log.open(StringIO.new)
|
247
|
+
expect { @store.sync }.to raise_error(PEROBS::FatalError)
|
248
|
+
PEROBS.log.open($stderr)
|
240
249
|
end
|
241
250
|
|
242
251
|
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/FixedSizeBlobFile'
|
30
|
+
|
31
|
+
describe PEROBS::FixedSizeBlobFile do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('FixedSizeBlobFile')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@bf = PEROBS::FixedSizeBlobFile.new(@db_dir, 'FixedSizeBlobFile', 4)
|
37
|
+
end
|
38
|
+
|
39
|
+
after(:all) do
|
40
|
+
FileUtils.rm_rf(@db_dir)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should create the file' do
|
44
|
+
@bf.open
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'should return free addresses' do
|
48
|
+
expect(@bf.free_address).to eql(0)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should store and retrieve a blob' do
|
52
|
+
@bf.store_blob(0,'0000')
|
53
|
+
expect(@bf.retrieve_blob(0)).to eql('0000')
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should store and retrieve multiple blobs' do
|
57
|
+
@bf.store_blob(0,'XXXX')
|
58
|
+
@bf.store_blob(1,'1111')
|
59
|
+
@bf.store_blob(2,'2222')
|
60
|
+
@bf.store_blob(3,'3333')
|
61
|
+
expect(@bf.retrieve_blob(0)).to eql('XXXX')
|
62
|
+
expect(@bf.retrieve_blob(1)).to eql('1111')
|
63
|
+
expect(@bf.retrieve_blob(2)).to eql('2222')
|
64
|
+
expect(@bf.retrieve_blob(3)).to eql('3333')
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should return nil for a too large address' do
|
68
|
+
expect(@bf.retrieve_blob(4)).to be_nil
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should delete an entry' do
|
72
|
+
@bf.delete_blob(2)
|
73
|
+
expect(@bf.retrieve_blob(4)).to be_nil
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'should return 2 as an empty address now' do
|
77
|
+
expect(@bf.free_address).to eql(2)
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'should persist all values over and close/open' do
|
81
|
+
@bf.close
|
82
|
+
@bf.open
|
83
|
+
|
84
|
+
expect(@bf.retrieve_blob(0)).to eql('XXXX')
|
85
|
+
expect(@bf.retrieve_blob(1)).to eql('1111')
|
86
|
+
expect(@bf.retrieve_blob(2)).to be_nil
|
87
|
+
expect(@bf.retrieve_blob(3)).to eql('3333')
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/FlatFileDB'
|
30
|
+
|
31
|
+
describe PEROBS::FlatFileDB do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('FlatFileDB')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@db = PEROBS::FlatFileDB.new(@db_dir)
|
37
|
+
@db.open
|
38
|
+
end
|
39
|
+
|
40
|
+
after(:each) do
|
41
|
+
@db.check_db
|
42
|
+
end
|
43
|
+
|
44
|
+
after(:all) do
|
45
|
+
@db.close
|
46
|
+
FileUtils.rm_rf(@db_dir)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should have a version file' do
|
50
|
+
version_file = File.join(@db_dir, 'version')
|
51
|
+
expect(File.exist?(version_file)).to be true
|
52
|
+
expect(File.read(version_file).to_i).to eq(PEROBS::FlatFileDB::VERSION)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|