perobs 2.3.1 → 2.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/perobs/Array.rb +17 -4
- data/lib/perobs/BTreeBlob.rb +22 -23
- data/lib/perobs/BTreeDB.rb +11 -12
- data/lib/perobs/Cache.rb +5 -4
- data/lib/perobs/DataBase.rb +19 -7
- data/lib/perobs/DynamoDB.rb +1 -1
- data/lib/perobs/FixedSizeBlobFile.rb +189 -0
- data/lib/perobs/FlatFile.rb +513 -0
- data/lib/perobs/FlatFileDB.rb +249 -0
- data/lib/perobs/FreeSpaceManager.rb +204 -0
- data/lib/perobs/Hash.rb +17 -4
- data/lib/perobs/IndexTree.rb +164 -0
- data/lib/perobs/IndexTreeNode.rb +296 -0
- data/lib/perobs/Log.rb +125 -0
- data/lib/perobs/Object.rb +10 -11
- data/lib/perobs/ObjectBase.rb +18 -5
- data/lib/perobs/StackFile.rb +137 -0
- data/lib/perobs/Store.rb +85 -19
- data/lib/perobs/TreeDB.rb +276 -0
- data/lib/perobs/version.rb +1 -1
- data/test/Array_spec.rb +11 -2
- data/test/FixedSizeBlobFile_spec.rb +91 -0
- data/test/FlatFileDB_spec.rb +56 -0
- data/test/FreeSpaceManager_spec.rb +91 -0
- data/test/Hash_spec.rb +11 -2
- data/test/IndexTree_spec.rb +118 -0
- data/test/Object_spec.rb +29 -17
- data/test/StackFile_spec.rb +113 -0
- data/test/Store_spec.rb +37 -3
- metadata +22 -3
data/lib/perobs/Store.rb
CHANGED
@@ -28,10 +28,12 @@
|
|
28
28
|
require 'set'
|
29
29
|
require 'weakref'
|
30
30
|
|
31
|
+
require 'perobs/Log'
|
31
32
|
require 'perobs/Handle'
|
32
33
|
require 'perobs/Cache'
|
33
34
|
require 'perobs/ClassMap'
|
34
35
|
require 'perobs/BTreeDB'
|
36
|
+
require 'perobs/FlatFileDB'
|
35
37
|
require 'perobs/Object'
|
36
38
|
require 'perobs/Hash'
|
37
39
|
require 'perobs/Array'
|
@@ -100,9 +102,9 @@ module PEROBS
|
|
100
102
|
# @param options [Hash] various options to affect the operation of the
|
101
103
|
# database. Currently the following options are supported:
|
102
104
|
# :engine : The class that provides the back-end storage
|
103
|
-
# engine. By default
|
105
|
+
# engine. By default FlatFileDB is used. A user
|
104
106
|
# can provide it's own storage engine that must
|
105
|
-
# conform to the same API exposed by
|
107
|
+
# conform to the same API exposed by FlatFileDB.
|
106
108
|
# :cache_bits : the number of bits used for cache indexing. The
|
107
109
|
# cache will hold 2 to the power of bits number of
|
108
110
|
# objects. We have separate caches for reading and
|
@@ -126,6 +128,7 @@ module PEROBS
|
|
126
128
|
def initialize(data_base, options = {})
|
127
129
|
# Create a backing store handler
|
128
130
|
@db = (options[:engine] || BTreeDB).new(data_base, options)
|
131
|
+
@db.open
|
129
132
|
# Create a map that can translate classes to numerical IDs and vice
|
130
133
|
# versa.
|
131
134
|
@class_map = ClassMap.new(@db)
|
@@ -143,13 +146,60 @@ module PEROBS
|
|
143
146
|
|
144
147
|
# The named (global) objects IDs hashed by their name
|
145
148
|
unless (@root_objects = object_by_id(0))
|
149
|
+
PEROBS.log.debug "Initializing the PEROBS store"
|
146
150
|
# The root object hash always has the object ID 0.
|
147
151
|
@root_objects = _construct_po(Hash, 0)
|
148
152
|
# Mark the root_objects object as modified.
|
149
153
|
@cache.cache_write(@root_objects)
|
150
154
|
end
|
155
|
+
unless @root_objects.is_a?(Hash)
|
156
|
+
PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
|
157
|
+
"but is a #{@root_objects.class}"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
# Copy the store content into a new Store. The arguments are identical to
|
162
|
+
# Store.new().
|
163
|
+
# @param data_base [String] the name of the database
|
164
|
+
# @param options [Hash] various options to affect the operation of the
|
165
|
+
def copy(dir, options = {})
|
166
|
+
# Make sure all objects are persisted.
|
167
|
+
sync
|
168
|
+
|
169
|
+
# Create a new store with the specified directory and options.
|
170
|
+
new_db = Store.new(dir, options)
|
171
|
+
# Clear the cache.
|
172
|
+
new_db.sync
|
173
|
+
# Copy all objects of the existing store to the new store.
|
174
|
+
i = 0
|
175
|
+
each do |ref_obj|
|
176
|
+
obj = ref_obj._referenced_object
|
177
|
+
obj._transfer(new_db)
|
178
|
+
obj._sync
|
179
|
+
i += 1
|
180
|
+
end
|
181
|
+
PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
|
182
|
+
# Flush the new store and close it.
|
183
|
+
new_db.exit
|
184
|
+
|
185
|
+
true
|
186
|
+
end
|
187
|
+
|
188
|
+
|
189
|
+
# Close the store and ensure that all in-memory objects are written out to
|
190
|
+
# the storage backend. The Store object is no longer usable after this
|
191
|
+
# method was called.
|
192
|
+
def exit
|
193
|
+
if @cache.in_transaction?
|
194
|
+
PEROBS.log.fatal 'You cannot call exit() during a transaction'
|
195
|
+
end
|
196
|
+
@cache.flush
|
197
|
+
@db.close
|
198
|
+
@db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
|
199
|
+
nil
|
151
200
|
end
|
152
201
|
|
202
|
+
|
153
203
|
# You need to call this method to create new PEROBS objects that belong to
|
154
204
|
# this Store.
|
155
205
|
# @param klass [Class] The class of the object you want to create. This
|
@@ -159,7 +209,7 @@ module PEROBS
|
|
159
209
|
# @return [POXReference] A reference to the newly created object.
|
160
210
|
def new(klass, *args)
|
161
211
|
unless klass.is_a?(BasicObject)
|
162
|
-
|
212
|
+
PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
|
163
213
|
end
|
164
214
|
|
165
215
|
obj = _construct_po(klass, _new_id, *args)
|
@@ -204,12 +254,12 @@ module PEROBS
|
|
204
254
|
# We only allow derivatives of PEROBS::Object to be stored in the
|
205
255
|
# store.
|
206
256
|
unless obj.is_a?(ObjectBase)
|
207
|
-
|
208
|
-
|
257
|
+
PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
|
258
|
+
"is of class #{obj.class}"
|
209
259
|
end
|
210
260
|
|
211
261
|
unless obj.store == self
|
212
|
-
|
262
|
+
PEROBS.log.fatal 'The object does not belong to this store.'
|
213
263
|
end
|
214
264
|
|
215
265
|
# Store the name and mark the name list as modified.
|
@@ -229,11 +279,17 @@ module PEROBS
|
|
229
279
|
POXReference.new(self, id)
|
230
280
|
end
|
231
281
|
|
282
|
+
# Return a list with all the names of the root objects.
|
283
|
+
# @return [Array of Symbols]
|
284
|
+
def names
|
285
|
+
@root_objects.keys
|
286
|
+
end
|
287
|
+
|
232
288
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
233
289
|
# needed.
|
234
290
|
def sync
|
235
291
|
if @cache.in_transaction?
|
236
|
-
|
292
|
+
PEROBS.log.fatal 'You cannot call sync() during a transaction'
|
237
293
|
end
|
238
294
|
@cache.flush
|
239
295
|
end
|
@@ -245,7 +301,7 @@ module PEROBS
|
|
245
301
|
# @return [Fixnum] The number of collected objects
|
246
302
|
def gc
|
247
303
|
if @cache.in_transaction?
|
248
|
-
|
304
|
+
PEROBS.log.fatal 'You cannot call gc() during a transaction'
|
249
305
|
end
|
250
306
|
sync
|
251
307
|
mark
|
@@ -299,11 +355,21 @@ module PEROBS
|
|
299
355
|
@db.clear_marks
|
300
356
|
|
301
357
|
errors = 0
|
358
|
+
objects = 0
|
302
359
|
@root_objects.each do |name, id|
|
360
|
+
objects += 1
|
303
361
|
errors += check_object(id, repair)
|
304
362
|
end
|
363
|
+
if errors > 0
|
364
|
+
PEROBS.log.warn "#{errors} errors found in #{objects} objects"
|
365
|
+
else
|
366
|
+
PEROBS.log.debug "No errors found"
|
367
|
+
end
|
305
368
|
@root_objects.delete_if { |name, id| !@db.check(id, false) }
|
306
369
|
|
370
|
+
# Ensure that any fixes are written into the DB.
|
371
|
+
sync
|
372
|
+
|
307
373
|
errors
|
308
374
|
end
|
309
375
|
|
@@ -334,8 +400,8 @@ module PEROBS
|
|
334
400
|
while !stack.empty?
|
335
401
|
# Get an object index from the stack.
|
336
402
|
unless (obj = object_by_id(id = stack.pop))
|
337
|
-
|
338
|
-
|
403
|
+
PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
|
404
|
+
"not found."
|
339
405
|
end
|
340
406
|
# Mark the object so it will never be pushed to the stack again.
|
341
407
|
@db.mark(id)
|
@@ -352,6 +418,7 @@ module PEROBS
|
|
352
418
|
def rename_classes(rename_map)
|
353
419
|
@class_map.rename(rename_map)
|
354
420
|
end
|
421
|
+
|
355
422
|
# Internal method. Don't use this outside of this library!
|
356
423
|
# Generate a new unique ID that is not used by any other object. It uses
|
357
424
|
# random numbers between 0 and 2**64 - 1.
|
@@ -379,12 +446,10 @@ module PEROBS
|
|
379
446
|
end
|
380
447
|
|
381
448
|
# Remove the object from the in-memory list. This is an internal method
|
382
|
-
# and should never be called from user code.
|
449
|
+
# and should never be called from user code. It will be called from a
|
450
|
+
# finalizer, so many restrictions apply!
|
383
451
|
# @param id [Fixnum or Bignum] Object ID of object to remove from the list
|
384
452
|
def _collect(id, ignore_errors = false)
|
385
|
-
unless ignore_errors || @in_memory_objects.include?(id)
|
386
|
-
raise RuntimeError, "Object with id #{id} is currently not in memory"
|
387
|
-
end
|
388
453
|
@in_memory_objects.delete(id)
|
389
454
|
end
|
390
455
|
|
@@ -408,6 +473,7 @@ module PEROBS
|
|
408
473
|
|
409
474
|
# The root_objects object is included in the count, but we only want to
|
410
475
|
# count user objects here.
|
476
|
+
PEROBS.log.debug "#{marked_objects - 1} objects marked"
|
411
477
|
@stats.marked_objects = marked_objects - 1
|
412
478
|
end
|
413
479
|
|
@@ -416,6 +482,7 @@ module PEROBS
|
|
416
482
|
def sweep
|
417
483
|
@stats.swept_objects = @db.delete_unmarked_objects.length
|
418
484
|
@cache.reset
|
485
|
+
PEROBS.log.debug "#{@stats.swept_objects} objects collected"
|
419
486
|
@stats.swept_objects
|
420
487
|
end
|
421
488
|
|
@@ -449,13 +516,12 @@ module PEROBS
|
|
449
516
|
else
|
450
517
|
# Remove references to bad objects.
|
451
518
|
if ref_obj && repair
|
452
|
-
|
453
|
-
|
519
|
+
PEROBS.log.warn "Fixing broken reference to object #{id} in " +
|
520
|
+
"object #{ref_obj._id}:\n" + ref_obj.inspect
|
454
521
|
ref_obj._delete_reference_to_id(id)
|
455
522
|
else
|
456
|
-
|
457
|
-
|
458
|
-
ref_obj.inspect
|
523
|
+
PEROBS.log.fatal "The following object references a " +
|
524
|
+
"non-existing object #{id}:\n" + ref_obj.inspect
|
459
525
|
end
|
460
526
|
errors += 1
|
461
527
|
end
|
@@ -0,0 +1,276 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# = BTreeDB.rb -- Persistent Ruby Object Store
|
4
|
+
#
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
+
#
|
7
|
+
# MIT License
|
8
|
+
#
|
9
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
+
# a copy of this software and associated documentation files (the
|
11
|
+
# "Software"), to deal in the Software without restriction, including
|
12
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
+
# the following conditions:
|
16
|
+
#
|
17
|
+
# The above copyright notice and this permission notice shall be
|
18
|
+
# included in all copies or substantial portions of the Software.
|
19
|
+
#
|
20
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
+
|
28
|
+
require 'fileutils'
|
29
|
+
|
30
|
+
require 'perobs/Log'
|
31
|
+
require 'perobs/DataBase'
|
32
|
+
require 'perobs/BTreeBlob'
|
33
|
+
|
34
|
+
module PEROBS
|
35
|
+
|
36
|
+
# This class implements a BTree database using filesystem directories as
|
37
|
+
# nodes and blob files as leafs. The BTree grows with the number of stored
|
38
|
+
# entries. Each leaf node blob can hold a fixed number of entries. If more
|
39
|
+
# entries need to be stored, the blob is replaced by a node with multiple
|
40
|
+
# new leafs that store the entries of the previous node. The leafs are
|
41
|
+
# implemented by the BTreeBlob class.
|
42
|
+
class BTreeDB < DataBase
|
43
|
+
|
44
|
+
attr_reader :max_blob_size
|
45
|
+
|
46
|
+
# Create a new BTreeDB object.
|
47
|
+
# @param db_name [String] name of the DB directory
|
48
|
+
# @param options [Hash] options to customize the behavior. Currently only
|
49
|
+
# the following options are supported:
|
50
|
+
# :serializer : Can be :marshal, :json, :yaml
|
51
|
+
# :dir_bits : The number of bits to use for the BTree nodes.
|
52
|
+
# The value must be between 4 and 14. The larger
|
53
|
+
# the number the more back-end directories are
|
54
|
+
# being used. The default is 12 which results in
|
55
|
+
# 4096 directories per node.
|
56
|
+
# :max_blob_size : The maximum number of entries in the BTree leaf
|
57
|
+
# nodes. The insert/find/delete time grows
|
58
|
+
# linearly with the size.
|
59
|
+
def initialize(db_name, options = {})
|
60
|
+
super(options[:serializer] || :json)
|
61
|
+
|
62
|
+
@db_dir = db_name
|
63
|
+
# Create the database directory if it doesn't exist yet.
|
64
|
+
ensure_dir_exists(@db_dir)
|
65
|
+
|
66
|
+
# Read the existing DB config.
|
67
|
+
@config = get_hash('config')
|
68
|
+
check_option('serializer')
|
69
|
+
|
70
|
+
# Check and set @dir_bits, the number of bits used for each tree level.
|
71
|
+
@dir_bits = options[:dir_bits] || 12
|
72
|
+
if @dir_bits < 4 || @dir_bits > 14
|
73
|
+
PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
|
74
|
+
"and 12"
|
75
|
+
end
|
76
|
+
check_option('dir_bits')
|
77
|
+
|
78
|
+
@max_blob_size = options[:max_blob_size] || 32
|
79
|
+
if @max_blob_size < 4 || @max_blob_size > 128
|
80
|
+
PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
|
81
|
+
"between 4 and 128"
|
82
|
+
end
|
83
|
+
check_option('max_blob_size')
|
84
|
+
|
85
|
+
put_hash('config', @config)
|
86
|
+
|
87
|
+
# This format string is used to create the directory name.
|
88
|
+
@dir_format_string = "%0#{(@dir_bits / 4) +
|
89
|
+
(@dir_bits % 4 == 0 ? 0 : 1)}X"
|
90
|
+
# Bit mask to extract the dir_bits LSBs.
|
91
|
+
@dir_mask = 2 ** @dir_bits - 1
|
92
|
+
end
|
93
|
+
|
94
|
+
# Delete the entire database. The database is no longer usable after this
|
95
|
+
# method was called.
|
96
|
+
def delete_database
|
97
|
+
FileUtils.rm_rf(@db_dir)
|
98
|
+
end
|
99
|
+
|
100
|
+
def BTreeDB::delete_db(db_name)
|
101
|
+
FileUtils.rm_rf(db_name)
|
102
|
+
end
|
103
|
+
|
104
|
+
# Return true if the object with given ID exists
|
105
|
+
# @param id [Fixnum or Bignum]
|
106
|
+
def include?(id)
|
107
|
+
!(blob = find_blob(id)).nil? && !blob.find(id).nil?
|
108
|
+
end
|
109
|
+
|
110
|
+
# Store a simple Hash as a JSON encoded file into the DB directory.
|
111
|
+
# @param name [String] Name of the hash. Will be used as file name.
|
112
|
+
# @param hash [Hash] A Hash that maps String objects to strings or
|
113
|
+
# numbers.
|
114
|
+
def put_hash(name, hash)
|
115
|
+
file_name = File.join(@db_dir, name + '.json')
|
116
|
+
begin
|
117
|
+
File.write(file_name, hash.to_json)
|
118
|
+
rescue => e
|
119
|
+
PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
# Load the Hash with the given name.
|
124
|
+
# @param name [String] Name of the hash.
|
125
|
+
# @return [Hash] A Hash that maps String objects to strings or numbers.
|
126
|
+
def get_hash(name)
|
127
|
+
file_name = File.join(@db_dir, name + '.json')
|
128
|
+
return ::Hash.new unless File.exist?(file_name)
|
129
|
+
|
130
|
+
begin
|
131
|
+
json = File.read(file_name)
|
132
|
+
rescue => e
|
133
|
+
PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
|
134
|
+
end
|
135
|
+
JSON.parse(json, :create_additions => true)
|
136
|
+
end
|
137
|
+
|
138
|
+
# Store the given object into the cluster files.
|
139
|
+
# @param obj [Hash] Object as defined by PEROBS::ObjectBase
|
140
|
+
def put_object(obj, id)
|
141
|
+
find_blob(id, true).write_object(id, serialize(obj))
|
142
|
+
end
|
143
|
+
|
144
|
+
# Load the given object from the filesystem.
|
145
|
+
# @param id [Fixnum or Bignum] object ID
|
146
|
+
# @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
|
147
|
+
# not exist
|
148
|
+
def get_object(id)
|
149
|
+
return nil unless (blob = find_blob(id)) && (obj = blob.read_object(id))
|
150
|
+
deserialize(obj)
|
151
|
+
end
|
152
|
+
|
153
|
+
# This method must be called to initiate the marking process.
|
154
|
+
def clear_marks
|
155
|
+
each_blob { |blob| blob.clear_marks }
|
156
|
+
end
|
157
|
+
|
158
|
+
# Permanently delete all objects that have not been marked. Those are
|
159
|
+
# orphaned and are no longer referenced by any actively used object.
|
160
|
+
# @return [Array] List of IDs that have been removed from the DB.
|
161
|
+
def delete_unmarked_objects
|
162
|
+
deleted_ids = []
|
163
|
+
each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
|
164
|
+
deleted_ids
|
165
|
+
end
|
166
|
+
|
167
|
+
# Mark an object.
|
168
|
+
# @param id [Fixnum or Bignum] ID of the object to mark
|
169
|
+
def mark(id)
|
170
|
+
(blob = find_blob(id)) && blob.mark(id)
|
171
|
+
end
|
172
|
+
|
173
|
+
# Check if the object is marked.
|
174
|
+
# @param id [Fixnum or Bignum] ID of the object to check
|
175
|
+
# @param ignore_errors [Boolean] If set to true no errors will be raised
|
176
|
+
# for non-existing objects.
|
177
|
+
def is_marked?(id, ignore_errors = false)
|
178
|
+
(blob = find_blob(id)) && blob.is_marked?(id, ignore_errors)
|
179
|
+
end
|
180
|
+
|
181
|
+
# Basic consistency check.
|
182
|
+
# @param repair [TrueClass/FalseClass] True if found errors should be
|
183
|
+
# repaired.
|
184
|
+
def check_db(repair = false)
|
185
|
+
each_blob { |blob| blob.check(repair) }
|
186
|
+
end
|
187
|
+
|
188
|
+
# Check if the stored object is syntactically correct.
|
189
|
+
# @param id [Fixnum/Bignum] Object ID
|
190
|
+
# @param repair [TrueClass/FalseClass] True if an repair attempt should be
|
191
|
+
# made.
|
192
|
+
# @return [TrueClass/FalseClass] True if the object is OK, otherwise
|
193
|
+
# false.
|
194
|
+
def check(id, repair)
|
195
|
+
begin
|
196
|
+
get_object(id)
|
197
|
+
rescue => e
|
198
|
+
PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
|
199
|
+
return false
|
200
|
+
end
|
201
|
+
|
202
|
+
true
|
203
|
+
end
|
204
|
+
|
205
|
+
# Store the given serialized object into the cluster files. This method is
|
206
|
+
# for internal use only!
|
207
|
+
# @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
|
208
|
+
# @param id [Fixnum or Bignum] Object ID
|
209
|
+
def put_raw_object(raw, id)
|
210
|
+
find_blob(id, true).write_object(id, raw)
|
211
|
+
end
|
212
|
+
|
213
|
+
private
|
214
|
+
|
215
|
+
def find_blob(id, create_missing_blob = false, dir_name = @db_dir)
|
216
|
+
dir_bits = id & @dir_mask
|
217
|
+
sub_dir_name = File.join(dir_name, @dir_format_string % dir_bits)
|
218
|
+
|
219
|
+
if Dir.exist?(sub_dir_name)
|
220
|
+
if File.exist?(File.join(sub_dir_name, 'index'))
|
221
|
+
# The directory is a blob directory and not a BTree node dir.
|
222
|
+
return BTreeBlob.new(sub_dir_name, self)
|
223
|
+
end
|
224
|
+
else
|
225
|
+
Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
|
226
|
+
# Extract the 01-part of the filename
|
227
|
+
lsb_string = File.basename(fqfn)[0..-6]
|
228
|
+
# Convert the lsb_string into a Fixnum
|
229
|
+
lsb = Integer('0b' + lsb_string)
|
230
|
+
# Bit mask to match the LSBs
|
231
|
+
mask = (2 ** lsb_string.length) - 1
|
232
|
+
if (id & mask) == lsb
|
233
|
+
return TreeBlob.new(sub_dir_name, lsb_string, self)
|
234
|
+
end
|
235
|
+
end
|
236
|
+
if create_missing_blob
|
237
|
+
# Create the new blob directory.
|
238
|
+
Dir.mkdir(dir_name)
|
239
|
+
# And initialize the blob DB.
|
240
|
+
return BTreeBlob.new(dir_name, self)
|
241
|
+
else
|
242
|
+
return nil
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
# Discard the least significant @dir_bits bits and start over again
|
247
|
+
# with the directory that matches the @dir_bits LSBs of the new ID.
|
248
|
+
id = id >> @dir_bits
|
249
|
+
end
|
250
|
+
|
251
|
+
def each_blob(&block)
|
252
|
+
each_blob_r(@db_dir, &block)
|
253
|
+
end
|
254
|
+
|
255
|
+
def each_blob_r(dir, &block)
|
256
|
+
Dir.glob(File.join(dir, '*')) do |dir_name|
|
257
|
+
if is_blob_dir?(dir_name)
|
258
|
+
block.call(BTreeBlob.new(dir_name, self))
|
259
|
+
else
|
260
|
+
each_blob_r(dir_name, &block)
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def is_blob_dir?(dir_name)
|
266
|
+
# A blob directory contains an 'index' and 'data' file. This is in
|
267
|
+
# contrast to BTree node directories that only contain other
|
268
|
+
# directories.
|
269
|
+
index_file = File.join(dir_name, 'index')
|
270
|
+
File.exist?(index_file)
|
271
|
+
end
|
272
|
+
|
273
|
+
end
|
274
|
+
|
275
|
+
end
|
276
|
+
|
data/lib/perobs/version.rb
CHANGED
data/test/Array_spec.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
3
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
4
|
#
|
5
5
|
# This file contains tests for Array that are similar to the tests for the
|
6
6
|
# Array implementation in MRI. The ideas of these tests were replicated in
|
@@ -224,6 +224,13 @@ describe PEROBS::Array do
|
|
224
224
|
pcheck { expect(a).to eq([ 1, 2, 3, 4, 5, nil ]) }
|
225
225
|
end
|
226
226
|
|
227
|
+
it 'should support inspect' do
|
228
|
+
a1 = cpa([ 1 ])
|
229
|
+
a2 = cpa([ 1, a1 ])
|
230
|
+
expect(a1.inspect).to eq("<PEROBS::Array:#{a1._id}>\n[\n 1\n]\n")
|
231
|
+
expect(a2.inspect).to eq("<PEROBS::Array:#{a2._id}>\n[\n 1,\n <PEROBS::ObjectBase:#{a1._id}>\n]\n")
|
232
|
+
end
|
233
|
+
|
227
234
|
it 'should only provide POXReference objects' do
|
228
235
|
a = cpa([ @store.new(PO), @store.new(PO) ])
|
229
236
|
expect(a[0].respond_to?(:is_poxreference?)).to be true
|
@@ -236,7 +243,9 @@ describe PEROBS::Array do
|
|
236
243
|
@store['a'] = a = @store.new(PEROBS::Array)
|
237
244
|
o = @store.new(PO)
|
238
245
|
a[0] = o.get_self
|
239
|
-
|
246
|
+
PEROBS.log.open(StringIO.new)
|
247
|
+
expect { @store.sync }.to raise_error(PEROBS::FatalError)
|
248
|
+
PEROBS.log.open($stderr)
|
240
249
|
end
|
241
250
|
|
242
251
|
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/FixedSizeBlobFile'
|
30
|
+
|
31
|
+
describe PEROBS::FixedSizeBlobFile do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('FixedSizeBlobFile')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@bf = PEROBS::FixedSizeBlobFile.new(@db_dir, 'FixedSizeBlobFile', 4)
|
37
|
+
end
|
38
|
+
|
39
|
+
after(:all) do
|
40
|
+
FileUtils.rm_rf(@db_dir)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should create the file' do
|
44
|
+
@bf.open
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'should return free addresses' do
|
48
|
+
expect(@bf.free_address).to eql(0)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should store and retrieve a blob' do
|
52
|
+
@bf.store_blob(0,'0000')
|
53
|
+
expect(@bf.retrieve_blob(0)).to eql('0000')
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should store and retrieve multiple blobs' do
|
57
|
+
@bf.store_blob(0,'XXXX')
|
58
|
+
@bf.store_blob(1,'1111')
|
59
|
+
@bf.store_blob(2,'2222')
|
60
|
+
@bf.store_blob(3,'3333')
|
61
|
+
expect(@bf.retrieve_blob(0)).to eql('XXXX')
|
62
|
+
expect(@bf.retrieve_blob(1)).to eql('1111')
|
63
|
+
expect(@bf.retrieve_blob(2)).to eql('2222')
|
64
|
+
expect(@bf.retrieve_blob(3)).to eql('3333')
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should return nil for a too large address' do
|
68
|
+
expect(@bf.retrieve_blob(4)).to be_nil
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should delete an entry' do
|
72
|
+
@bf.delete_blob(2)
|
73
|
+
expect(@bf.retrieve_blob(4)).to be_nil
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'should return 2 as an empty address now' do
|
77
|
+
expect(@bf.free_address).to eql(2)
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'should persist all values over and close/open' do
|
81
|
+
@bf.close
|
82
|
+
@bf.open
|
83
|
+
|
84
|
+
expect(@bf.retrieve_blob(0)).to eql('XXXX')
|
85
|
+
expect(@bf.retrieve_blob(1)).to eql('1111')
|
86
|
+
expect(@bf.retrieve_blob(2)).to be_nil
|
87
|
+
expect(@bf.retrieve_blob(3)).to eql('3333')
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/FlatFileDB'
|
30
|
+
|
31
|
+
describe PEROBS::FlatFileDB do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('FlatFileDB')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@db = PEROBS::FlatFileDB.new(@db_dir)
|
37
|
+
@db.open
|
38
|
+
end
|
39
|
+
|
40
|
+
after(:each) do
|
41
|
+
@db.check_db
|
42
|
+
end
|
43
|
+
|
44
|
+
after(:all) do
|
45
|
+
@db.close
|
46
|
+
FileUtils.rm_rf(@db_dir)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should have a version file' do
|
50
|
+
version_file = File.join(@db_dir, 'version')
|
51
|
+
expect(File.exist?(version_file)).to be true
|
52
|
+
expect(File.read(version_file).to_i).to eq(PEROBS::FlatFileDB::VERSION)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|